Subject: port-sparc64/30371: ALTQ loses packets on sparc64 only
To: None <port-sparc64-maintainer@netbsd.org, gnats-admin@netbsd.org,>
From: None <carton@Ivy.NET>
List: netbsd-bugs
Date: 05/29/2005 23:51:00
>Number:         30371
>Category:       port-sparc64
>Synopsis:       ALTQ loses packets on sparc64 only
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    port-sparc64-maintainer
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Sun May 29 23:51:00 +0000 2005
>Originator:     Miles Nordin
>Release:        NetBSD 2.0.2_STABLE + PF/ALTQ patch from pflkm 20050118
>Organization:
Ivy Ministries
>Environment:
System: NetBSD lucette 2.0.2_STABLE NetBSD 2.0.2_STABLE (LUCETTE-$Revision: 1.3 $) #0: Sat May 28 15:42:58 EDT 2005 carton@castrovalva:/scratch/src/sys/arch/sparc64/compile/LUCETTE sparc64
Architecture: sparc64
Machine: sparc64
>Description:
	
>How-To-Repeat:
get NetBSD 2.0.2 sources
'make patch' for security/pflkm 2005-01-18
apply work/pflkm-20050118/patches/altq.diff to the kernel sources
  SHA1 (pflkm/20050118/altq.diff) = 45b930cb9bf27ae9dce1be9910864f214ed27ebc
  RMD160 (pflkm/20050118/altq.diff) = 49d3e3cbc16b9a5a8e6560f56d48f50c67ac5378
  Size (pflkm/20050118/altq.diff) = 387410 bytes
note: this is the same altq.diff that's in pflkm-20050511
apply work/pflkm-20050118/patches/if_events.diff

build a kernel.  build pflkm with PKG_OPTIONS.pf= ifevents altq

add to /etc/lkm.conf
-----8<-----
/usr/pkg/lkm/pf.o 	-	-	-		-	AFTERMOUNT
-----8<-----

add to /etc/rc.conf
-----8<-----
lkm=YES
pf=YES
-----8<-----

I tried to make this simpler in two ways:

 * reduce the queue structure to just two subqueues instead of 40
   -- couldn't reproduce the problem

 * include just the queue structure, no classification rules
   -- couldn't reproduce the problem

so, apparently to see the problem you must have several queues, you must 
have packets in more than just one of them, and you must have some 
meaningful traffic flowing.

add to /etc/pf/pf.conf
-----8<-----
# Macros

# XXX -- change $innurnet4 in /etc/pf/spoof/innurnet
innurnet4  = gem0
innurnet6  = gif0
inside     = gre0

# everything else (including everything non-TCP) to <ptpuser4> is 
# considered peer-to-peer
wkport	= "{ www, pop3, imap, finger, netbios-ssn, silc, afpovertcp, https, \
	     imaps, 6667, whois, 5190, 8000, 8010 }"

natsrc	     = "69.31.131.39"

# downstream ALTQ
shardy		= "2001:4830:2150:41:209:5bff:fe1f:9b0"
shardy4		= "69.31.131.50"
phar		= "2001:4830:2150:41:20c:76ff:fe5c:2c1b"
#ircserver4	= "69.31.131.XXX"
#ircserver	= "2001:4830:2150:41:XXX"
irchub4		= "209.58.245.42"
antioch4	= "69.31.131.48"
kish4		= "192.168.1.122"
akkad4		= "192.168.1.119"
setient		= "2001:4830:2150:41:a00:7ff:fea6:fdd2"
setient4	= "69.31.131.55"
mangala4	= "69.31.131.41"
heidi4		= "192.168.15.26"
listor4		= "192.168.1.101"
gregor		= "2001:4830:2150:1:2a0:ccff:fe32:b8c4"
gregor4		= "192.168.1.121"
zaphod		= "2001:4830:2150:1:210:b5ff:fe5b:6dd"
zaphod4		= "192.168.1.100"
esquivel	= "2001:4830:2150:41:210:5aff:fea7:e8"
esquivel4	= "69.31.131.60"
euclid4		= "192.168.1.108"

live365		= "216.235.81.0/24"


# Tables

#new referenced by downstream
table <phar4>        { 69.31.131.44, 69.31.131.45, $euclid4, \
		       192.168.1.113, 192.168.15.18, 192.168.3.42, \
		       192.168.3.94 }
table <carton>	     { 2001:4830:2150:41:a00:2bff:fe86:af46, \
		       2001:4830:2150:c0::3, \
		       2001:4830:2150:c0::5 }
table <carton4>	     { 69.31.131.61, 192.168.15.14 }
table <souterrain>   { 2001:4830:2150:41:a00:20ff:fe11:f5b3 }
table <souterrain4>  { $akkad4, $kish4, $antioch4, 192.168.15.34 }
table <hhh4-wireless> \
		     { 192.168.15.0/24, 192.168.16.0/20, 192.168.32.0/20 }
table <b9punk>	     { $gregor, $zaphod }
table <ptpuser4>     { $listor4, $heidi4, $gregor4, $euclid4 }

#referenced by upstream/downstream
table <lucette4> { 69.31.131.34, 69.31.131.35 }
table <lucette> { 3ffe:401d:203a:c0::4, \
		   2001:4830:2150:c0::4, \
		   2001:4830:e2:11::2 }
table <ezln4> { 216.158.24.193, 216.158.24.222, 208.0.42.29, \
		    69.31.131.36, 69.31.131.37 }
table <ezln> {	3ffe:401d:203a:c0::1, \
			3ffe:401d:203a:1::1, \
			3ffe:401d:203a:3::1, \
			3ffe:401d:203a:41::1, \
			3ffe:401d:203a:42::1, \
			2001:4830:2150:c0::1, \
			2001:4830:2150:1::1, \
			2001:4830:2150:3::1, \
			2001:4830:2150:41::1, \
			2001:4830:2150:42::1 }

#referenced by non-QoS stuff too
table <rfc1918> { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }
table <hhh4-private> { 192.168.0.0/16 } # will be a subset of 10/8 for Innernet
table <hhh4-global> { 69.31.131.32/27 }
table <beyondfw4> { 69.31.131.33/32 }
# from rfc3330
# 0.0.0.0/8		link-local? 0.0.0.0 for DHCP?
# 169.254.0.0/16	link-local
# 192.0.2.0/24		documentation examples
# 224.0.0.0/4		multicast
# 240.0.0.0/4		reserved
# 192.88.99.0/24	everyone's closest tunnel is Microsoft
table <notunicast4> { 224.0.0.0/3, 0.0.0.0/8, 169.254.0.0/16, 192.0.2.0/24 }
table <bogus4> { 127.0.0.0/8, 240.0.0.0/4, 192.0.2.0/24, 169.254.0.0/16 }

table <hhh-private> { 3ffe:401d:203a:0::/58, 2001:4830:2150:0::/58 }
table <hhh-public> { 3ffe:401d:203a:40::/58, 3ffe:401d:203a:c0::/64, 2001:4830:2150:40::/58, 2001:4830:2150:c0::/64 }
table <hhh>   { 3ffe:401d:203a::/48, 2001:4830:2150::/48, 2001:4830:e2:11::2/128  }
table <hhhll> { 3ffe:401d:203a::/48, 2001:4830:2150::/48, 2001:4830:e2:11::2/128, fe80::/10 }
# from rfc2373
# 0::/8			reserved
# fe80::/10		link-local
# fec0::/10		site-local
# ff::/8		multicast (should never appear in source address)
# 0:0:0:0:0:ffff::/96	ipv4-mapped-ipv6 for API
# 0:0:0:0:0:0::/96	nonsensical ipv4 autotunnel
# 2::/7			NSAP
# 4::/7			IPX
table <notunicast> { 0::/8, 2::/7, 4::/7, fec0::/10, ff::/8 }
table <bogus> { 0::/8, 2::/7, 4::/7, fec0::/10 }

# /etc/pf/blackhole should contain DDoS participants, and optionally
# members of irc AKILL lists, one IP address per line with # comments
# allowed.  Add IPs to that file, not to pf.conf.
table <blackhole4> persist file "/etc/pf/blackhole4"
table <blackhole> persist file "/etc/pf/blackhole6"

# Options

set timeout tcp.established 90000 # 25 hours
# this adaptive.end should yeild 70min established timeout at 10000 states.
# other timeouts than tcp.established may not be reasonable, though.
set timeout { adaptive.start 6000, adaptive.end 10918 }
# src-nodes is unused, I think
set limit { states 10000, frags 10000, src-nodes 2000 }
set state-policy if-bound
set block-policy drop

# Scrub

#goddamnit.  wtf is wrong with this.  maybe try 'no-df'.
#scrub in on $outside all fragment reassemble

scrub out on $innurnet4 inet from <hhh4-global> to any max-mss 1436

# ALTQ

# lucette: downstream link-sharing
# ezln: upstream link-sharing

#this is lucette
#
# method 1:
#  run altq on gem0 ($innurnet4) with an 'upperlimit' subqueue, and 
#  classify packets entering gre0 ($inside) so they go to leaves 
#  underneath this 'upperlimit'ed subqueue.  the 'queue' tags will 
#  stay on the packets when gre adds tunnel headers to them.
#
# method 2:
#  run altq on ppp0 ($inside) with a token buffer matching downstream 
#  capacity.  These packets will get shoved inside an ssh session.  no 
#  ALTQ on the ssh session, so downstream will be a little choppy.
#
# method 3:
#  ipsec tunnel mode and NAT traversal.  not sure if queue tags will 
#  survive ipsec encapsulation, but they probably do.
#

#method1
altq on $innurnet4 hfsc bandwidth 768000b \
	queue { hhh-downstream, pilosoft-tr }

# the default queue will get things PF is unable to classify with just
# its hook in ip_output.c, like ARP.
queue pilosoft-tr bandwidth 384000b hfsc( default )

queue hhh-downstream  bandwidth   384000b hfsc \
	{ guest, ptp, people }

queue guest        bandwidth       10%               { guest-b, guest-rt }
    queue guest-b      bandwidth       60% hfsc( ecn )
    queue guest-rt     bandwidth       40% hfsc( ecn )
queue ptp          bandwidth       30%                   { ptp-b, ptp-rt }
    queue ptp-b        bandwidth       50% qlimit 100 hfsc( ecn )
# we use RED/ECN most places, but not on queues that should never be 
# dropping packets, like this one which contains only TCP ACKs, and on 
# the VoIP and irc queues.
    queue ptp-rt       bandwidth       50% qlimit 60 hfsc

queue people       bandwidth       60% \
	{ lastresort, router, other, \
	  esquivel, carton, carton-irc, sout, sout-voip, phar, shardy, \
	  setient, mangala, lucas, heidi, b9punk }
  queue lastresort   bandwidth      6.5% hfsc
  queue router       bandwidth      3.5% hfsc
  queue other        bandwidth      4.8%               { other-b, other-rt }
    queue other-b      bandwidth       60% hfsc( ecn )
    queue other-rt     bandwidth       40% hfsc( ecn )

  queue esquivel     bandwidth      4.8%         { esquivel-b, esquivel-rt }
    queue esquivel-b   bandwidth       60% hfsc( ecn )
    queue esquivel-rt  bandwidth       40% hfsc( ecn )
  queue carton       bandwidth      4.8%             { carton-b, carton-rt }
    queue carton-b     bandwidth       60% hfsc( ecn )
    queue carton-rt    bandwidth       40% hfsc( ecn )
# irc is reflector, so upstream is fatter than downstream.
# should use realtime here.
    queue carton-irc bandwidth      4.8%   hfsc( ecn )
  queue sout         bandwidth      4.8%                 { sout-b, sout-rt }
    queue sout-b       bandwidth       60% hfsc( ecn )
    queue sout-rt      bandwidth       40% hfsc( ecn )
# should use realtime here.  voip doesn't need a giant chunk of the link.
# it needs some fixed bandwidth.
    queue sout-voip  bandwidth       17%   hfsc(       \
						     linkshare 17% )
  queue phar         bandwidth      4.8%                 { phar-b, phar-rt }
    queue phar-b       bandwidth       60% hfsc( ecn )
    queue phar-rt      bandwidth       40% hfsc( ecn )
  queue shardy       bandwidth      4.8%             { shardy-b, shardy-rt }
    queue shardy-b     bandwidth       60% hfsc( ecn )
    queue shardy-rt    bandwidth       40% hfsc( ecn )
  queue setient      bandwidth      4.8%           { setient-b, setient-rt }
    queue setient-b    bandwidth       60% hfsc( ecn )
    queue setient-rt   bandwidth       40% hfsc( ecn )
  queue mangala      bandwidth      4.8%           { mangala-b, mangala-rt }
    queue mangala-b    bandwidth       60% hfsc( ecn )
    queue mangala-rt   bandwidth       40% hfsc( ecn )
  queue lucas        bandwidth      4.8%           { lucas-b, lucas-rt }
    queue lucas-b      bandwidth       60% hfsc( ecn )
    queue lucas-rt     bandwidth       40% hfsc( ecn )
  queue heidi        bandwidth      4.8%               { heidi-b, heidi-rt }
    queue heidi-b      bandwidth       60% hfsc( ecn )
    queue heidi-rt     bandwidth       40% hfsc( ecn )
  queue b9punk       bandwidth      4.8%             { b9punk-b, b9punk-rt }
    queue b9punk-b     bandwidth       60% hfsc( ecn )
    queue b9punk-rt    bandwidth       40% hfsc( ecn )

# NAT

# don't use FTP proxy for stuff headed to lucette
no rdr on $inside inet proto tcp from <hhh4-private> to <hhh4-global> port 21

# ftp will appear to the ftp server to originate from an address on the 
# router's interface, while other NAT will originate from an address not 
# assigned to any interface, the nat.hackerhalfwayhouse.org and such.
# therefore, we can't use ftp-proxy's '-n' option.
rdr on $inside inet proto tcp from <hhh4-private> to ! <hhh4-private> port 21 \
	-> 192.168.168.4 port 8021

# UDP rdr as we do it differs in two ways from UDP NAT
#  1. hosts on the outside are allowed to send the first UDP, and create 
#     a state association for a few seconds.  (10?  20?)  With regular NAT 
#     UDP, the inside host must send the first packet (``outgoing only'').
#  2. the inside and outside port number is fixed---for us, they are both
#     the same number.  With regular outgoing-only NAT, the outside port 
#     number is allocated dynamically.
# to implement this we need both an rdr rule and a NAT rule.  You should 
# not see a rdr...proto udp without a nat...proto udp to go with it.
#
# if you use 'keep state' rules to go with these (on lucette, we
# don't), remember filter rules happen after translation, which means
# translated inside address for 'rdr' and translated outside address
# for 'nat'.  or, each translation rule needs both 'keep state' rules
# for the backflow?....  i'm confused.  Anyway I added two 'keep
# state' like this:
#rdr on $innurnet4 inet proto udp from any to $natsrc port 4569 \
#	-> $kish4 port 4569
#nat on $innurnet4 inet proto udp from $kish4 port 4569 to any -> \
#	$natsrc port 4569
#pass in  quick on $innurnet4 inet proto udp \
#	from any to $kish4 port 4569 \
#	keep state queue inside-voip
#pass out quick on $innurnet4 inet proto udp \
#	from $natsrc port 4569 to any \
#	keep state queue inside-voip
# probably it is better to use 'nat pass' and 'rdr pass', but I didn't 
# think of that the last time I did it.

# this is edonkey
rdr on $innurnet4 inet proto tcp from any to $natsrc port 4661:4662 \
	-> $listor4 port 4661:*
rdr on $innurnet4 inet proto udp from any to $natsrc port 4672 \
	-> $listor4 port 4672
nat on $innurnet4 inet proto udp from $listor4 port 4672 to any -> \
	$natsrc port 4672
#
rdr on $innurnet4 inet proto tcp from any to $natsrc port 14662 \
	-> $akkad4 port 14662
rdr on $innurnet4 inet proto udp from any to $natsrc port 14672 \
	-> $akkad4 port 14672
nat on $innurnet4 inet proto udp from $akkad4 port 14672 to any -> \
	$natsrc port 14672
#
rdr on $innurnet4 inet proto tcp from any to $natsrc port 24662 \
	-> $gregor4 port 24662
rdr on $innurnet4 inet proto udp from any to $natsrc port 24672 \
	-> $gregor4 port 24672
nat on $innurnet4 inet proto udp from $gregor4 port 24672 to any -> \
	$natsrc port 24672


# this is bittorrent
rdr on $innurnet4 inet proto tcp from any to $natsrc port 13137 \
	-> $euclid4 port 13137

# this is IAX VoIP but...i think it's not working anymore?  this rule
# may need changing.
rdr on $innurnet4 inet proto udp from any to $natsrc port 4569 \
	-> $kish4 port 4569
nat on $innurnet4 inet proto udp from $kish4 port 4569 to any -> \
	$natsrc port 4569

# now for traditional outgoing nat
# we used to use 'nat pass' which means translated packets get through 
# the filter automatically, but now there is no security boundary here 
# on lucette so just 'nat' without 'pass' will do.
#this should mean ICMP related to UDP and TCP flows, too.
nat on $innurnet4 inet from <hhh4-private> to any -> $natsrc

# Filter

#filter abuse based on source address
# say ``net-prohib'' rather than ``host-prohib,'' even though from our
# perspective we are blocking a host, since most ICMP refers to the
# destination, not the source, with the presumption the packet sender
# should have no discretion to change the source.  In this case, the
# sender is prohibited access to an entire net, so net-prohib feels 
# right.  anyway, the difference should be only to humans looking at it, 
# not to well-written TCP stacks that, to avoid ICMP attacks against TCP, 
# must strictly look at the TCP header included in the ICMP and the fact 
# that it's ``unreachable,'' but...who knows what will happen out there.
#
# the reason we don't send these is that currently there is no rule to
# match and no implicit match of PF_TAG_PF_GENERATED packets, so the
# ICMP we generate here will get re-run through this ruleset and
# reclassified, so there's no way for us to stuff it into a special
# rate-limited queue separate from any other ICMP-unreachable.  The
# other problem is that right now NetBSD's pfil_hooks->PF wrapper is
# causing the regular IP stack to send an ICMP unreachable for any
# 'block drop', so if we ask for one, then two get sent.  need to fix
# that.
#
#block return-icmp(net-prohib) in quick on $innurnet4 inet \
#	from <blackhole4> to any queue admin-icmp
#block return-icmp6(admin-unr) in quick on $innurnet6 inet6 \
#	from <blackhole> to any queue admin-icmp
block drop in quick on $innurnet4 inet \
	from <blackhole4> to any queue admin-icmp
block drop in quick on $innurnet6 inet6 \
	from <blackhole> to any queue admin-icmp
#no point in returning ICMP to an unreachable destination.
#but, this loose-uRPF stuff is broken in NetBSD.
#block in log quick inet from no-route to any
#block in log quick inet6 from no-route to any


# anti-spoofing

# this is really important, because we make this assumption throughout 
# the rules without even thinking about it.
block in log quick on $innurnet4 inet6 from any to any
block in log quick on $innurnet6 inet from any to any

# note that multicast packets have unicast source addresses, always.
pass in quick on lo0 inet from 127.0.0.1 to 127.0.0.1
pass in quick on lo0 inet6 from ::1 to ::1
pass out quick on lo0 inet from 127.0.0.1 to 127.0.0.1
pass out quick on lo0 inet6 from ::1 to ::1
block out log quick inet from <notunicast4> to any
block out log quick inet6 from <notunicast> to any
block in log quick inet from <bogus4> to any
block in log quick inet6 from <bogus> to any
block in log quick inet from any to <bogus4>
block in log quick inet6 from any to <bogus>

# in PF, firewall is applied after NAT.
block out log quick on $innurnet4 inet  from ! <hhh4-global> to any
block out log quick on $innurnet6 inet6 from ! <hhh> to any

block in log quick on $innurnet4 inet from <rfc1918> to any
anchor "spoof/innurnet" in on $innurnet4 inet from ! <beyondfw4> to any
load anchor "spoof/innurnet4" from "/etc/pf/spoof/innurnet4"
 #block in log quick on $innurnet4 inet from <hhh4-global> to any
block in log quick on $innurnet6 inet6 from <hhh> to any

block in log quick on { $innurnet4, $innurnet6 } proto ospf

# ICMP section
## ICMP types for reference
#static const struct icmptypeent icmp_type[] = {
#        { "echoreq",    ICMP_ECHO },
#        { "echorep",    ICMP_ECHOREPLY },
#        { "unreach",    ICMP_UNREACH },
#        { "squench",    ICMP_SOURCEQUENCH },
#        { "redir",      ICMP_REDIRECT },
#        { "althost",    ICMP_ALTHOSTADDR },
#        { "routeradv",  ICMP_ROUTERADVERT },
#        { "routersol",  ICMP_ROUTERSOLICIT },
#        { "timex",      ICMP_TIMXCEED },
#        { "paramprob",  ICMP_PARAMPROB },
#        { "timereq",    ICMP_TSTAMP },
#        { "timerep",    ICMP_TSTAMPREPLY },
#        { "inforeq",    ICMP_IREQ },
#        { "inforep",    ICMP_IREQREPLY },
#        { "maskreq",    ICMP_MASKREQ },
#        { "maskrep",    ICMP_MASKREPLY },
#        { "trace",      ICMP_TRACEROUTE },
#        { "dataconv",   ICMP_DATACONVERR },
#        { "mobredir",   ICMP_MOBILE_REDIRECT },
#        { "ipv6-where", ICMP_IPV6_WHEREAREYOU },
#        { "ipv6-here",  ICMP_IPV6_IAMHERE },
#        { "mobregreq",  ICMP_MOBILE_REGREQUEST },
#        { "mobregrep",  ICMP_MOBILE_REGREPLY },
#        { "skip",       ICMP_SKIP },
#        { "photuris",   ICMP_PHOTURIS }
#};
#
## ICMP codes
#        { "net-unr",            ICMP_UNREACH,   ICMP_UNREACH_NET },
#        { "host-unr",           ICMP_UNREACH,   ICMP_UNREACH_HOST },
#        { "proto-unr",          ICMP_UNREACH,   ICMP_UNREACH_PROTOCOL },
#        { "port-unr",           ICMP_UNREACH,   ICMP_UNREACH_PORT },
#        { "needfrag",           ICMP_UNREACH,   ICMP_UNREACH_NEEDFRAG },
#        { "srcfail",            ICMP_UNREACH,   ICMP_UNREACH_SRCFAIL },
#        { "net-unk",            ICMP_UNREACH,   ICMP_UNREACH_NET_UNKNOWN },
#        { "host-unk",           ICMP_UNREACH,   ICMP_UNREACH_HOST_UNKNOWN },
#        { "isolate",            ICMP_UNREACH,   ICMP_UNREACH_ISOLATED },
#        { "net-prohib",         ICMP_UNREACH,   ICMP_UNREACH_NET_PROHIB },
#        { "host-prohib",        ICMP_UNREACH,   ICMP_UNREACH_HOST_PROHIB },
#        { "net-tos",            ICMP_UNREACH,   ICMP_UNREACH_TOSNET },
#        { "host-tos",           ICMP_UNREACH,   ICMP_UNREACH_TOSHOST },
#        { "filter-prohib",      ICMP_UNREACH,   ICMP_UNREACH_FILTER_PROHIB },
#        { "host-preced",        ICMP_UNREACH,   ICMP_UNREACH_HOST_PRECEDENCE },
#        { "cutoff-preced",      ICMP_UNREACH,   ICMP_UNREACH_PRECEDENCE_CUTOFF }
#,
#        { "redir-net",          ICMP_REDIRECT,  ICMP_REDIRECT_NET },
#        { "redir-host",         ICMP_REDIRECT,  ICMP_REDIRECT_HOST },
#        { "redir-tos-net",      ICMP_REDIRECT,  ICMP_REDIRECT_TOSNET },
#        { "redir-tos-host",     ICMP_REDIRECT,  ICMP_REDIRECT_TOSHOST },
#        { "normal-adv",         ICMP_ROUTERADVERT, ICMP_ROUTERADVERT_NORMAL },
#        { "common-adv",         ICMP_ROUTERADVERT, ICMP_ROUTERADVERT_NOROUTE_COM
#MON },
#        { "transit",            ICMP_TIMXCEED,  ICMP_TIMXCEED_INTRANS },
#        { "reassemb",           ICMP_TIMXCEED,  ICMP_TIMXCEED_REASS },
#        { "badhead",            ICMP_PARAMPROB, ICMP_PARAMPROB_ERRATPTR },
#        { "optmiss",            ICMP_PARAMPROB, ICMP_PARAMPROB_OPTABSENT },
#        { "badlen",             ICMP_PARAMPROB, ICMP_PARAMPROB_LENGTH },
#        { "unknown-ind",        ICMP_PHOTURIS,  ICMP_PHOTURIS_UNKNOWN_INDEX },
#        { "auth-fail",          ICMP_PHOTURIS,  ICMP_PHOTURIS_AUTH_FAILED },
#        { "decrypt-fail",       ICMP_PHOTURIS,  ICMP_PHOTURIS_DECRYPT_FAILED }
#
## ICMPv6 types
#static const struct icmptypeent icmp6_type[] = {
#        { "unreach",    ICMP6_DST_UNREACH },
#        { "toobig",     ICMP6_PACKET_TOO_BIG },
#        { "timex",      ICMP6_TIME_EXCEEDED },
#        { "paramprob",  ICMP6_PARAM_PROB },
#        { "echoreq",    ICMP6_ECHO_REQUEST },
#        { "echorep",    ICMP6_ECHO_REPLY },
#        { "groupqry",   ICMP6_MEMBERSHIP_QUERY },
#        { "listqry",    MLD_LISTENER_QUERY },
#        { "grouprep",   ICMP6_MEMBERSHIP_REPORT },
#        { "listenrep",  MLD_LISTENER_REPORT },
#        { "groupterm",  ICMP6_MEMBERSHIP_REDUCTION },
#        { "listendone", MLD_LISTENER_DONE },
#        { "routersol",  ND_ROUTER_SOLICIT },
#        { "routeradv",  ND_ROUTER_ADVERT },
#        { "neighbrsol", ND_NEIGHBOR_SOLICIT },
#        { "neighbradv", ND_NEIGHBOR_ADVERT },
#        { "redir",      ND_REDIRECT },
#        { "routrrenum", ICMP6_ROUTER_RENUMBERING },
#        { "wrureq",     ICMP6_WRUREQUEST },
#        { "wrurep",     ICMP6_WRUREPLY },
#        { "fqdnreq",    ICMP6_FQDN_QUERY },
#        { "fqdnrep",    ICMP6_FQDN_REPLY },
#        { "niqry",      ICMP6_NI_QUERY },
#        { "nirep",      ICMP6_NI_REPLY },
#        { "mtraceresp", MLD_MTRACE_RESP },
#        { "mtrace",     MLD_MTRACE }
#};
#
## ICMPv6 codes
#static const struct icmpcodeent icmp6_code[] = {
#        { "admin-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_ADMIN },
#        { "noroute-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOROUTE },
#        { "notnbr-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOTNEIGHBOR },
#        { "beyond-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_BEYONDSCOPE },
#        { "addr-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_ADDR },
#        { "port-unr", ICMP6_DST_UNREACH, ICMP6_DST_UNREACH_NOPORT },
#        { "transit", ICMP6_TIME_EXCEEDED, ICMP6_TIME_EXCEED_TRANSIT },
#        { "reassemb", ICMP6_TIME_EXCEEDED, ICMP6_TIME_EXCEED_REASSEMBLY },
#        { "badhead", ICMP6_PARAM_PROB, ICMP6_PARAMPROB_HEADER },
#        { "nxthdr", ICMP6_PARAM_PROB, ICMP6_PARAMPROB_NEXTHEADER },
#        { "redironlink", ND_REDIRECT, ND_REDIRECT_ONLINK },
#        { "redirrouter", ND_REDIRECT, ND_REDIRECT_ROUTER }
#};

block drop in log quick inet proto icmp all icmp-type redir queue admin-icmp
block drop in log quick inet6 proto ipv6-icmp all icmp6-type redir queue admin-icmp
#
# code 4 is needed for PMTU and will break things in confusing and
# embarrassing ways if blocked.
# however, the need to pass it is only for hosts not covered by
# outgoing-only firewall because stateful inspection will pass
# relevant ICMP.
pass in quick on $innurnet4 inet proto icmp from any to <hhh4-global> \
	icmp-type unreach code needfrag
#
# bad port.  for UDP equiv to ``connection refused''.
pass in quick on $innurnet4 inet proto icmp from any to <hhh4-global> \
	icmp-type unreach code port-unr
#
# proto, net, host unreachable
pass in quick on $innurnet4 inet proto icmp from any to <hhh4-global> \
	icmp-type { unreach code net-unr, \
		    unreach code host-unr, \
		    unreach code proto-unr }
#
# admin. prohib.
pass in quick on $innurnet4 inet proto icmp from any to <hhh4-global> \
	icmp-type { unreach code net-prohib, \
		    unreach code host-prohib, \
		    unreach code filter-prohib }
#
# for testing ALTQ
pass in quick on $innurnet4 proto icmp from 216.158.25.1 to <hhh4-global> \
	icmp-type echorep
pass in quick on $innurnet4 inet proto icmp from 207.245.113.23 to 208.0.42.17 \
	icmp-type echoreq
#
# for traceroute
pass in quick on $innurnet4 inet proto icmp from any to <hhh4-global> \
	icmp-type timex queue router
#block drop in log quick on $innurnet4 proto icmp from any to any \
#	queue admin-icmp
# end ICMP section


# classifiers section
#  this is lucette, so these are for inbound, downstream on the 
#  T1/sshppp/IPsec or whatever

# this is redundant, i think, now that there's no problem of ``LAN'' 
# packets between lucette and the shelf.
#pass out on $inside inet proto tcp from any to any user proxy \
#	queue ( other-b, other-rt )

# here is the order of rules and how they cast shadows on each other
#  remember case of phar, for example, where some of his IPs are inside 
#  <ptpuser4> and some aren't.
# 1. inet from any to any queue other
#    inet6 ...
# 2. inet proto tcp from any to any queue other
#    inet6 ...
#
# 3. inet from any to <person> queue person
#    inet6 ...
# 4. omitted if user is all-ptp
#       inet proto tcp from any to <person> queue person
# 5. inet6 ...#4...
#
# ---- section barrier
# 6. inet from any to <ptpuser4> queue ptp
# ---- section barrier
#
# 7. inet proto tcp from any port $wkport to <ptpuser4> queue other
# 8. inet proto tcp from $live365 to any queue other-rt
# 9. inet proto tcp from any port { 6667, ssh } to any queue other-rt
#    inet6 ...
#    inet proto tcp from any to <person> port ssh queue other-rt
#    inet6 ...
#
# 10.omitted if user is all-nonptp
#       inet proto tcp from any port $wkport to <person> queue person
# 11.inet proto tcp from any port { 6667, ssh } to <person> queue person-rt
#    inet6 ...
# 12.omitted if user has nothing on shelf
#    inet proto tcp from any to <person> port ssh queue person-rt
#    inet6 ...
# 13.inet proto tcp from $live365 to <person> queue person-rt

#1
#this rule is 'log' because most of what it catches is a mistake.
pass out log on $inside all \
	queue other-b

#2
pass out on $inside inet6 proto tcp from any to any \
	queue ( other-b, other-rt )
pass out on $inside inet proto tcp from any to any \
	queue ( other-b, other-rt )

#3
pass out on $inside inet from any to <hhh4-wireless> \
	queue guest-b
pass out on $inside inet from any to 192.168.15.0/24 \
	queue other-b
pass out on $inside inet6 from any to $shardy \
	queue shardy-b
pass out on $inside inet from any to $shardy4 \
	queue shardy-b
pass out on $inside inet6 from any to $phar \
	queue phar-b
pass out on $inside inet from any to <phar4> \
	queue phar-b
pass out on $inside inet6 from any to <carton> \
	queue carton-b
pass out on $inside inet from any to <carton4> \
	queue carton-b
pass out on $inside inet6 from any to <souterrain> \
	queue sout-b
pass out on $inside inet from any to <souterrain4> \
	queue sout-b
pass out on $inside inet6 from any to $setient \
	queue setient-b
pass out on $inside inet from any to $setient4 \
	queue setient-b
pass out on $inside inet from any to $mangala4 \
	queue mangala-b
pass out on $inside inet from any to $heidi4 \
	queue heidi-b
pass out on $inside inet from any to $listor4 \
	queue lucas-b
pass out on $inside inet6 from any to <b9punk> \
	queue b9punk-b
pass out on $inside inet from any to <b9punk4> \
	queue b9punk-b
pass out on $inside inet6 from any to $esquivel \
	queue esquivel-b
pass out on $inside inet from any to $esquivel4 \
	queue esquivel-b

#4, #5
pass out on $inside inet proto tcp from any to <hhh4-wireless> \
	queue ( guest-b, guest-rt )
pass out on $inside inet proto tcp from any to 192.168.15.0/24 \
	queue ( other-b, other-rt )
pass out on $inside inet6 proto tcp from any to $shardy \
	queue ( shardy-b, shardy-rt )
pass out on $inside inet proto tcp from any to $shardy4 \
	queue ( shardy-b, shardy-rt )
pass out on $inside inet6 proto tcp from any to $phar \
	queue ( phar-b, phar-rt )
pass out on $inside inet proto tcp from any to <phar4> \
	queue ( phar-b, phar-rt )
pass out on $inside inet6 proto tcp from any to <carton> \
	queue ( carton-b, carton-rt )
pass out on $inside inet proto tcp from any to <carton4> \
	queue ( carton-b, carton-rt )
pass out on $inside inet6 proto tcp from any to <souterrain> \
	queue ( sout-b, sout-rt )
pass out on $inside inet proto tcp from any to <souterrain4> \
	queue ( sout-b, sout-rt )
pass out on $inside inet6 proto tcp from any to $setient \
	queue ( setient-b, setient-rt )
pass out on $inside inet proto tcp from any to $setient4 \
	queue ( setient-b, setient-rt )
pass out on $inside inet proto tcp from any to $mangala4 \
	queue ( mangala-b, mangala-rt )
pass out on $inside inet6 proto tcp from any to <b9punk> \
	queue ( b9punk-b, b9punk-rt )
pass out on $inside inet6 proto tcp from any to $esquivel \
	queue ( esquivel-b, esquivel-rt )
pass out on $inside inet proto tcp from any to $esquivel4 \
	queue ( esquivel-b, esquivel-rt )

#6
pass out on $inside inet from any to <ptpuser4> \
	queue ptp-b
pass out on $inside inet proto tcp from any to <ptpuser4> \
	queue ( ptp-b, ptp-rt )
#7
pass out on $inside inet proto tcp from any port $wkport to <ptpuser4> \
	queue ( other-b, other-rt )
#8
pass out on $inside inet proto tcp from $live365 to any \
	queue other-rt
#9
#some day I would like to test replacing this with something like
# pass in on $inside proto tcp from any port ssh to any \
#	flags SA/SAFR tos lowdelay keep state \
#	queue other-rt
#the idea is that, on the innurnet tos bits get mangled, but if tos bits 
#have never left our AS (have been protected by a tunnel?), then we can 
#see the 'lowdelay' marker ssh leaves us to tell it has allocated a tty 
#for this session (meaning it's probably interactive, not scp)
#
pass out on $inside inet proto tcp from any port { ssh, 6667 } to any \
	queue other-rt
pass out on $inside inet proto tcp from any to any port ssh \
	queue other-rt
pass out on $inside inet6 proto tcp from any port { ssh, 6667 } to any \
	queue other-rt
pass out on $inside inet6 proto tcp from any to any port ssh \
	queue other-rt

#10
pass out on $inside inet proto tcp from any port $wkport to $listor4 \
	queue ( lucas-b, lucas-rt )
pass out on $inside inet proto tcp from any port $wkport to $heidi4 \
	queue ( heidi-b, heidi-rt )
pass out on $inside inet proto tcp from any port $wkport to $gregor4 \
	queue ( b9punk-b, b9punk-rt )
pass out on $inside inet proto tcp from any port $wkport to $euclid4 \
	queue ( phar-b, phar-rt )

#11, 12
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to <hhh4-wireless> \
	queue guest-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to 192.168.15.0/24 \
	queue other-rt
pass out on $inside inet6 proto tcp from any to $shardy port ssh \
	queue shardy-rt
pass out on $inside inet6 proto tcp from any port { ssh, 6667 } to $shardy \
	queue shardy-rt
pass out on $inside inet proto tcp from any to $shardy4 port ssh \
	queue shardy-rt
pass out on $inside inet proto tcp from any port { ssh, 6667 } to $shardy4 \
	queue shardy-rt
pass out on $inside inet6 proto tcp from any to $phar port ssh \
	queue phar-rt
pass out on $inside inet6 proto tcp from any port { ssh, 6667 } to $phar \
	queue phar-rt
pass out on $inside inet proto tcp from any to <phar4> port ssh \
	queue phar-rt
pass out on $inside inet proto tcp from any port { ssh, 6667 } to <phar4> \
	queue phar-rt
pass out on $inside inet6 proto tcp from any to <carton> port ssh \
	queue carton-rt
pass out on $inside inet6 proto tcp \
	from any port { ssh, 6667 } to <carton> \
	queue carton-rt
pass out on $inside inet proto tcp from any to <carton4> port ssh \
	queue carton-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to <carton4> \
	queue carton-rt
#pass out on $inside inet6 proto tcp \
#	from any to $ircserver port { 6667, 706 } \
#	queue carton-irc
#pass out on $inside inet proto tcp \
#	from any to $ircserver4 port { 6667, 706 } \
#	queue carton-irc
#pass out on $inside inet proto tcp \
#	from $irchub4 port 6667 to $ircserver4 \
#	queue carton-irc
pass out on $inside inet6 proto tcp from any to <souterrain> port ssh \
	queue sout-rt
pass out on $inside inet6 proto tcp \
	from any port { ssh, 6667 } to <souterrain> \
	queue sout-rt
pass out on $inside inet proto tcp from any to <souterrain4> port ssh \
	queue sout-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to <souterrain4> \
	queue sout-rt
pass out on $inside inet6 proto tcp from any to $setient port ssh \
	queue setient-rt
pass out on $inside inet6 proto tcp \
	from any port { ssh, 6667 } to $setient \
	queue setient-rt
pass out on $inside inet proto tcp from any to $setient4 port ssh \
	queue setient-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to $setient4 \
	queue setient-rt
pass out on $inside inet proto tcp from any to $mangala4 port ssh \
	queue mangala-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to $mangala4 \
	queue mangala-rt
pass out on $inside inet proto tcp from any port { ssh, 6667 } to $heidi4 \
	queue heidi-rt
pass out on $inside inet proto tcp from any port { ssh, 6667 } to $listor4 \
	queue lucas-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to <b9punk4> \
	queue b9punk-rt
pass out on $inside inet6 proto tcp \
	from any port { ssh, 6667 } to <b9punk> \
	queue b9punk-rt
pass out on $inside inet6 proto tcp from any to $esquivel port ssh \
	queue esquivel-rt
pass out on $inside inet6 proto tcp \
	from any port { ssh, 6667 } to $esquivel \
	queue esquivel-rt
pass out on $inside inet proto tcp from any to $esquivel4 port ssh \
	queue esquivel-rt
pass out on $inside inet proto tcp \
	from any port { ssh, 6667 } to $esquivel4 \
	queue esquivel-rt

#13
pass out on $inside inet proto tcp from $live365 to <hhh4-wireless> \
	queue guest-rt
pass out on $inside inet proto tcp from $live365 to 192.168.15.0/24 \
	queue other-rt
pass out on $inside inet proto tcp from $live365 to $esquivel4 \
	queue esquivel-rt
pass out on $inside inet proto tcp from $live365 to <carton4> \
	queue carton-rt
pass out on $inside inet proto tcp from $live365 to <souterrain4> \
	queue sout-rt
pass out on $inside inet proto tcp from $live365 to <phar4> \
	queue phar-rt
pass out on $inside inet proto tcp from $live365 to $shardy4 \
	queue shardy-rt
pass out on $inside inet proto tcp from $live365 to $setient4 \
	queue setient-rt
pass out on $inside inet proto tcp from $live365 to $mangala4 \
	queue mangala-rt
pass out on $inside inet proto tcp from $live365 to $heidi4 \
	queue heidi-rt
pass out on $inside inet proto tcp from $live365 to $listor4 \
	queue lucas-rt
pass out on $inside inet proto tcp from $live365 to <b9punk4> \
	queue b9punk-rt

# the last word
pass out on $inside inet proto udp from any to $kish4 port 4569 \
	queue sout-voip

pass out on $inside inet6 from <lucette> to any \
	queue lastresort
pass out on $inside inet from <lucette4> to any \
	queue lastresort
pass out on $inside inet6 from any to <ezln> \
	queue router
pass out on $inside inet from any to <ezln4> \
	queue router
pass out on $inside inet6 from fe80::/10 to any \
	queue lastresort
pass out on $inside inet proto ospf from any to any \
	queue lastresort
-----8<-----

now, see the problem.  the address we're pinging here goes into the 
'pilosoft-tr' queue.
-----8<-----
lucette:/etc/pf$ netstat -nid | grep gem0.*Link
gem0  1500  <Link>        00:03:ba:0f:aa:45 23054647     0 23034811     0     0    58
lucette:~$ ping -c 100 69.31.131.33
[...]
----pilosoft-gw.Ivy.NET PING Statistics----
100 packets transmitted, 94 packets received, 6.0% packet loss
round-trip min/avg/max/stddev = 0.466/1.943/29.936/4.834 ms
lucette:/etc/pf$ netstat -nid | grep gem0.*Link
                            # only one dropped packet, while 6 pings are mising
gem0  1500  <Link>        00:03:ba:0f:aa:45 23097481     0 23077544     0     0    59
lucette:/etc/pf$ sudo pfctl -v -s queue # note the 'dropped' column is all zero
queue root_gem0 bandwidth 768Kb priority 0 {pilosoft-tr, hhh-downstream}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue  pilosoft-tr bandwidth 384Kb hfsc( default ) 
  [ pkts:      13902  bytes:    3109465  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue  hhh-downstream bandwidth 384Kb {guest, ptp, people}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue   guest bandwidth 38.40Kb {guest-b, guest-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    guest-b bandwidth 23.04Kb hfsc( red ecn ) 
  [ pkts:         12  bytes:       1512  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    guest-rt bandwidth 15.36Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue   ptp bandwidth 115.20Kb {ptp-b, ptp-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    ptp-b bandwidth 57.60Kb qlimit 100 hfsc( red ecn ) 
  [ pkts:       6732  bytes:    3917188  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/100 ]
queue    ptp-rt bandwidth 57.60Kb qlimit 60 
  [ pkts:       2099  bytes:     172122  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 60 ]
queue   people bandwidth 230.40Kb {lastresort, router, other, esquivel, carton, carton-irc, sout, sout-voip, phar, shardy, setient, mangala, lucas, heidi, b9punk}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    lastresort bandwidth 13.82Kb 
  [ pkts:        259  bytes:      68276  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    router bandwidth 6.91Kb 
  [ pkts:         34  bytes:       8098  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    other bandwidth 9.22Kb {other-b, other-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     other-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:         18  bytes:       6131  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     other-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:         26  bytes:      15267  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    esquivel bandwidth 9.22Kb {esquivel-b, esquivel-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     esquivel-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:         79  bytes:      15058  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     esquivel-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:       1664  bytes:     136552  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    carton bandwidth 9.22Kb {carton-b, carton-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     carton-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:        140  bytes:      23627  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     carton-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:         43  bytes:       6584  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    carton-irc bandwidth 9.22Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    sout bandwidth 9.22Kb {sout-b, sout-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     sout-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:          1  bytes:         86  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     sout-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    sout-voip bandwidth 39.17Kb 
  [ pkts:         44  bytes:       3948  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    phar bandwidth 9.22Kb {phar-b, phar-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     phar-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:       3066  bytes:    4524098  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     phar-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:        155  bytes:      17288  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    shardy bandwidth 9.22Kb {shardy-b, shardy-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     shardy-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     shardy-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    setient bandwidth 9.22Kb {setient-b, setient-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     setient-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:          3  bytes:        402  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     setient-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    mangala bandwidth 9.22Kb {mangala-b, mangala-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     mangala-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:          1  bytes:         86  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     mangala-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    lucas bandwidth 9.22Kb {lucas-b, lucas-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     lucas-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     lucas-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    heidi bandwidth 9.22Kb {heidi-b, heidi-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     heidi-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:        747  bytes:     222101  dropped pkts:      1 bytes:    203 ]
  [ qlength:   0/ 50 ]
queue     heidi-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:        112  bytes:      10196  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue    b9punk bandwidth 9.22Kb {b9punk-b, b9punk-rt}
  [ pkts:          0  bytes:          0  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     b9punk-b bandwidth 5.52Kb hfsc( red ecn ) 
  [ pkts:         35  bytes:       5851  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
queue     b9punk-rt bandwidth 3.68Kb hfsc( red ecn ) 
  [ pkts:         16  bytes:       1256  dropped pkts:      0 bytes:      0 ]
  [ qlength:   0/ 50 ]
lucette:/etc/pf$ sudo pfctl -F queue
altq cleared
lucette:~$ ping -c 100 69.31.131.33
[...]
----pilosoft-gw.Ivy.NET PING Statistics----
100 packets transmitted, 100 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 0.448/0.505/1.517/0.129 ms
-----8<-----

now, try all that all over again on an alpha, and find that it works ~fine.

>Fix:
not known.  It happens reliably with our traffic pattern, but I wish there 
were a simpler way to reproduce it.