--Kj7319i9nmIyA2yE
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

Folks...

This is the first part of a series of simple patches Wasabi Systems is
contributing that can substantially improve network performance on some
platforms.

The following patch adds support for caching the physical address of
pool objects allocated through the pool cache.  It adds two new functions
to the pool API:

	pool_cache_get_paddr()		Get an object from the pool
					cache, along with its physical
					address.

	pool_cache_put_paddr()		Put an object back to the pool
					cache, along with its physical
					address.

pool_cache_get() and pool_cache_put() become CPP wrappers around these new
calls.

Why would you want to do this?  This allows certain users of the pool
allocator to eliminate redundant work in some code paths.

Consider the case of sending and receiving data from the network.  Mbuf
clusters are allocated from the pool cache, and the physical address is
extracted from the kernel virtual address (and frobbed as necessary by
the bus_dma layer), and provided to the Ethernet controller.

Some profiling data that Allen Briggs gathered showed that the extraction
of the physical address accounted for a fair amount of work involved in
moving the data in and out of the network.  By caching the physical address
of both mbufs and clusters, the physical address extraction step for plain
mbufs and clusters can be avoided.

Any comments you might have on this patch are appreciated.  Note that
the next few patches I'll be posting rely on this one.

-- 
        -- Jason R. Thorpe <thorpej@wasabisystems.com>

--Kj7319i9nmIyA2yE
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename=pool-paddr-patch

Index: sys/sys/pool.h
===================================================================
RCS file: /cvsroot/src/sys/sys/pool.h,v
retrieving revision 1.38
diff -c -r1.38 pool.h
*** sys/sys/pool.h	2002/08/25 23:03:39	1.38
--- sys/sys/pool.h	2003/03/29 02:12:51
***************
*** 57,62 ****
--- 57,64 ----
  #define PR_HASHTABSIZE		8
  #define	PCG_NOBJECTS		16
  
+ #define	POOL_PADDR_INVALID	((paddr_t) -1)
+ 
  #ifdef __POOL_EXPOSE
  /* The pool cache group. */
  struct pool_cache_group {
***************
*** 64,70 ****
  		pcg_list;	/* link in the pool cache's group list */
  	u_int	pcg_avail;	/* # available objects */
  				/* pointers to the objects */
! 	void	*pcg_objects[PCG_NOBJECTS];
  };
  
  struct pool_cache {
--- 66,75 ----
  		pcg_list;	/* link in the pool cache's group list */
  	u_int	pcg_avail;	/* # available objects */
  				/* pointers to the objects */
! 	struct {
! 		void *pcgo_va;	/* cache object virtual address */
! 		paddr_t pcgo_pa;/* cache object physical address */
! 	} pcg_objects[PCG_NOBJECTS];
  };
  
  struct pool_cache {
***************
*** 250,257 ****
  		    void (*dtor)(void *, void *),
  		    void *);
  void		pool_cache_destroy(struct pool_cache *);
! void		*pool_cache_get(struct pool_cache *, int);
! void		pool_cache_put(struct pool_cache *, void *);
  void		pool_cache_destruct_object(struct pool_cache *, void *);
  void		pool_cache_invalidate(struct pool_cache *);
  #endif /* _KERNEL */
--- 255,265 ----
  		    void (*dtor)(void *, void *),
  		    void *);
  void		pool_cache_destroy(struct pool_cache *);
! void		*pool_cache_get_paddr(struct pool_cache *, int, paddr_t *);
! #define		pool_cache_get(pc, f) pool_cache_get_paddr((pc), (f), NULL)
! void		pool_cache_put_paddr(struct pool_cache *, void *, paddr_t);
! #define		pool_cache_put(pc, o) pool_cache_put_paddr((pc), (o), \
! 				          POOL_PADDR_INVALID)
  void		pool_cache_destruct_object(struct pool_cache *, void *);
  void		pool_cache_invalidate(struct pool_cache *);
  #endif /* _KERNEL */
Index: sys/kern/subr_pool.c
===================================================================
RCS file: /cvsroot/src/sys/kern/subr_pool.c,v
retrieving revision 1.86
diff -c -r1.86 subr_pool.c
*** sys/kern/subr_pool.c	2003/03/16 08:06:51	1.86
--- sys/kern/subr_pool.c	2003/03/29 02:12:52
***************
*** 1488,1495 ****
  		    pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);
  		TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
  			(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);
! 			for (i = 0; i < PCG_NOBJECTS; i++)
! 				(*pr)("\t\t\t%p\n", pcg->pcg_objects[i]);
  		}
  	}
  
--- 1488,1505 ----
  		    pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);
  		TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
  			(*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);
! 			for (i = 0; i < PCG_NOBJECTS; i++) {
! 				if (pcg->pcg_objects[i].pcgo_pa !=
! 				    POOL_PADDR_INVALID) {
! 					(*pr)("\t\t\t%p, 0x%llx\n",
! 					    pcg->pcg_objects[i].pcgo_va,
! 					    (unsigned long long)
! 					    pcg->pcg_objects[i].pcgo_pa);
! 				} else {
! 					(*pr)("\t\t\t%p\n",
! 					    pcg->pcg_objects[i].pcgo_va);
! 				}
! 			}
  		}
  	}
  
***************
*** 1618,1624 ****
  }
  
  static __inline void *
! pcg_get(struct pool_cache_group *pcg)
  {
  	void *object;
  	u_int idx;
--- 1628,1634 ----
  }
  
  static __inline void *
! pcg_get(struct pool_cache_group *pcg, paddr_t *pap)
  {
  	void *object;
  	u_int idx;
***************
*** 1627,1658 ****
  	KASSERT(pcg->pcg_avail != 0);
  	idx = --pcg->pcg_avail;
  
! 	KASSERT(pcg->pcg_objects[idx] != NULL);
! 	object = pcg->pcg_objects[idx];
! 	pcg->pcg_objects[idx] = NULL;
  
  	return (object);
  }
  
  static __inline void
! pcg_put(struct pool_cache_group *pcg, void *object)
  {
  	u_int idx;
  
  	KASSERT(pcg->pcg_avail < PCG_NOBJECTS);
  	idx = pcg->pcg_avail++;
  
! 	KASSERT(pcg->pcg_objects[idx] == NULL);
! 	pcg->pcg_objects[idx] = object;
  }
  
  /*
!  * pool_cache_get:
   *
!  *	Get an object from a pool cache.
   */
  void *
! pool_cache_get(struct pool_cache *pc, int flags)
  {
  	struct pool_cache_group *pcg;
  	void *object;
--- 1637,1672 ----
  	KASSERT(pcg->pcg_avail != 0);
  	idx = --pcg->pcg_avail;
  
! 	KASSERT(pcg->pcg_objects[idx].pcgo_va != NULL);
! 	object = pcg->pcg_objects[idx].pcgo_va;
! 	if (pap != NULL)
! 		*pap = pcg->pcg_objects[idx].pcgo_pa;
! 	pcg->pcg_objects[idx].pcgo_va = NULL;
  
  	return (object);
  }
  
  static __inline void
! pcg_put(struct pool_cache_group *pcg, void *object, paddr_t pa)
  {
  	u_int idx;
  
  	KASSERT(pcg->pcg_avail < PCG_NOBJECTS);
  	idx = pcg->pcg_avail++;
  
! 	KASSERT(pcg->pcg_objects[idx].pcgo_va == NULL);
! 	pcg->pcg_objects[idx].pcgo_va = object;
! 	pcg->pcg_objects[idx].pcgo_pa = pa;
  }
  
  /*
!  * pool_cache_get{,_paddr}:
   *
!  *	Get an object from a pool cache (optionally returning
!  *	the physical address of the object).
   */
  void *
! pool_cache_get_paddr(struct pool_cache *pc, int flags, paddr_t *pap)
  {
  	struct pool_cache_group *pcg;
  	void *object;
***************
*** 1687,1699 ****
  				return (NULL);
  			}
  		}
  		return (object);
  	}
  
   have_group:
  	pc->pc_hits++;
  	pc->pc_nitems--;
! 	object = pcg_get(pcg);
  
  	if (pcg->pcg_avail == 0)
  		pc->pc_allocfrom = NULL;
--- 1701,1720 ----
  				return (NULL);
  			}
  		}
+ 		if (object != NULL && pap != NULL) {
+ #ifdef POOL_VTOPHYS
+ 			*pap = POOL_VTOPHYS(object);
+ #else
+ 			*pap = POOL_PADDR_INVALID;
+ #endif
+ 		}
  		return (object);
  	}
  
   have_group:
  	pc->pc_hits++;
  	pc->pc_nitems--;
! 	object = pcg_get(pcg, pap);
  
  	if (pcg->pcg_avail == 0)
  		pc->pc_allocfrom = NULL;
***************
*** 1704,1715 ****
  }
  
  /*
!  * pool_cache_put:
   *
!  *	Put an object back to the pool cache.
   */
  void
! pool_cache_put(struct pool_cache *pc, void *object)
  {
  	struct pool_cache_group *pcg;
  	int s;
--- 1725,1737 ----
  }
  
  /*
!  * pool_cache_put{,_paddr}:
   *
!  *	Put an object back to the pool cache (optionally caching the
!  *	physical address of the object).
   */
  void
! pool_cache_put_paddr(struct pool_cache *pc, void *object, paddr_t pa)
  {
  	struct pool_cache_group *pcg;
  	int s;
***************
*** 1752,1758 ****
  
   have_group:
  	pc->pc_nitems++;
! 	pcg_put(pcg, object);
  
  	if (pcg->pcg_avail == PCG_NOBJECTS)
  		pc->pc_freeto = NULL;
--- 1774,1780 ----
  
   have_group:
  	pc->pc_nitems++;
! 	pcg_put(pcg, object, pa);
  
  	if (pcg->pcg_avail == PCG_NOBJECTS)
  		pc->pc_freeto = NULL;
***************
*** 1794,1800 ****
  		npcg = TAILQ_NEXT(pcg, pcg_list);
  		while (pcg->pcg_avail != 0) {
  			pc->pc_nitems--;
! 			object = pcg_get(pcg);
  			if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
  				pc->pc_allocfrom = NULL;
  			if (pc->pc_dtor != NULL)
--- 1816,1822 ----
  		npcg = TAILQ_NEXT(pcg, pcg_list);
  		while (pcg->pcg_avail != 0) {
  			pc->pc_nitems--;
! 			object = pcg_get(pcg, NULL);
  			if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
  				pc->pc_allocfrom = NULL;
  			if (pc->pc_dtor != NULL)
Index: sys/arch/alpha/include/pmap.h
===================================================================
RCS file: /cvsroot/src/sys/arch/alpha/include/pmap.h,v
retrieving revision 1.58
diff -c -r1.58 pmap.h
*** sys/arch/alpha/include/pmap.h	2003/01/17 22:11:16	1.58
--- sys/arch/alpha/include/pmap.h	2003/03/29 02:12:52
***************
*** 214,219 ****
--- 214,224 ----
  #define	PMAP_MAP_POOLPAGE(pa)		ALPHA_PHYS_TO_K0SEG((pa))
  #define	PMAP_UNMAP_POOLPAGE(va)		ALPHA_K0SEG_TO_PHYS((va))
  
+ /*
+  * Other hooks for the pool allocator.
+  */
+ #define	POOL_VTOPHYS(va)		ALPHA_K0SEG_TO_PHYS((va))
+ 
  boolean_t			pmap_pageidlezero(paddr_t);
  #define	PMAP_PAGEIDLEZERO(pa)	pmap_pageidlezero((pa))
  
Index: sys/arch/arm/include/arm32/pmap.h
===================================================================
RCS file: /cvsroot/src/sys/arch/arm/include/arm32/pmap.h,v
retrieving revision 1.63
diff -c -r1.63 pmap.h
*** sys/arch/arm/include/arm32/pmap.h	2003/03/23 15:59:24	1.63
--- sys/arch/arm/include/arm32/pmap.h	2003/03/29 02:12:53
***************
*** 1,7 ****
  /*	$NetBSD: pmap.h,v 1.63 2003/03/23 15:59:24 chris Exp $	*/
  
  /*
!  * Copyright (c 2002 Wasabi Systems, Inc.
   * All rights reserved.
   *
   * Written by Jason R. Thorpe for Wasabi Systems, Inc.
--- 1,7 ----
  /*	$NetBSD: pmap.h,v 1.63 2003/03/23 15:59:24 chris Exp $	*/
  
  /*
!  * Copyright (c) 2002 Wasabi Systems, Inc.
   * All rights reserved.
   *
   * Written by Jason R. Thorpe for Wasabi Systems, Inc.
***************
*** 440,445 ****
--- 440,451 ----
  #define	L2_S_PROT(ku, pr)	((((ku) == PTE_USER) ? L2_S_PROT_U : 0) | \
  				 (((pr) & VM_PROT_WRITE) ? L2_S_PROT_W : 0))
  
+ /*
+  * Hooks for the pool allocator.
+  */
+ #define	POOL_VTOPHYS(va)	vtophys((vaddr_t) (va))
+ 
  #endif /* _KERNEL */
  
  #endif	/* _ARM32_PMAP_H_ */
+ #endif	/* ARM32_PMAP_NEW */
Index: sys/arch/i386/include/pmap.h
===================================================================
RCS file: /cvsroot/src/sys/arch/i386/include/pmap.h,v
retrieving revision 1.70
diff -c -r1.70 pmap.h
*** sys/arch/i386/include/pmap.h	2003/03/03 22:13:16	1.70
--- sys/arch/i386/include/pmap.h	2003/03/29 02:12:53
***************
*** 504,508 ****
--- 504,513 ----
  #define	PMAP_FORK
  #endif /* USER_LDT */
  
+ /*
+  * Hooks for the pool allocator.
+  */
+ #define	POOL_VTOPHYS(va)	vtophys((vaddr_t) (va))
+ 
  #endif /* _KERNEL */
  #endif	/* _I386_PMAP_H_ */
Index: sys/arch/mips/include/pmap.h
===================================================================
RCS file: /cvsroot/src/sys/arch/mips/include/pmap.h,v
retrieving revision 1.41
diff -c -r1.41 pmap.h
*** sys/arch/mips/include/pmap.h	2002/11/30 01:52:32	1.41
--- sys/arch/mips/include/pmap.h	2003/03/29 02:12:53
***************
*** 152,157 ****
--- 152,162 ----
  #define	PMAP_UNMAP_POOLPAGE(va)	MIPS_KSEG0_TO_PHYS((va))
  
  /*
+  * Other hooks for the pool allocator.
+  */
+ #define	POOL_VTOPHYS(va)	MIPS_KSEG0_TO_PHYS((va))
+ 
+ /*
   * Select CCA to use for unmanaged pages.
   */
  #define	PMAP_CCA_FOR_PA(pa)	2		/* uncached */
Index: usr.bin/vmstat/vmstat.c
===================================================================
RCS file: /cvsroot/src/usr.bin/vmstat/vmstat.c,v
retrieving revision 1.111
diff -c -r1.111 vmstat.c
*** usr.bin/vmstat/vmstat.c	2003/03/19 11:36:35	1.111
--- usr.bin/vmstat/vmstat.c	2003/03/29 02:12:55
***************
*** 1139,1146 ****
  			    "pool cache group trashed");
  			printf("\t\tgroup %p: avail %d\n", pcg_addr,
  			    pcg->pcg_avail);
! 			for (i = 0; i < PCG_NOBJECTS; i++)
! 				printf("\t\t\t%p\n", pcg->pcg_objects[i]);
  		}
  	}
  
--- 1139,1156 ----
  			    "pool cache group trashed");
  			printf("\t\tgroup %p: avail %d\n", pcg_addr,
  			    pcg->pcg_avail);
! 			for (i = 0; i < PCG_NOBJECTS; i++) {
! 				if (pcg->pcg_objects[i].pcgo_pa !=
! 				    POOL_PADDR_INVALID) {
! 					printf("\t\t\t%p, 0x%llx\n",
! 					    pcg->pcg_objects[i].pcgo_va,
! 					    (unsigned long long)
! 					    pcg->pcg_objects[i].pcgo_pa);
! 				} else {
! 					printf("\t\t\t%p\n",
! 					    pcg->pcg_objects[i].pcgo_va);
! 				}
! 			}
  		}
  	}
  

--Kj7319i9nmIyA2yE--