I've recently done some studying of the MACHINE_NONCONTIG option
(and converted the sun3 port over to use it).  As a result, I
have some suggestions for improving the MACHINE_NONCONTIG code.

Introduction:

Many machines have a "mostly" contiguous physical memory space,
consisting of a large range (i.e. 16MB) marred by a small number
of unusable "holes".  For example, the i386/ISA/EISA bus machines
typically have about 16MB of memory which is contiguous except for
a the I/O hole at 640K..1M.  As another example, the Sun3/50 has
4MB (or 8MB) of contiguous memory, but 128K of it starting at 1M
is used for video RAM.

For machines like the above, with a few, small holes, it is
desirable to have the VM system define the vm_page_array with
holes in it that correspond directly to the real holes.
This allows the function pmap_page_index() to be fast (it
can be just a macro!) at the cost of a small amount of wasted
memory in the vm_page_array (10 ints per unused page).
This is the classic "speed vs. memory use" trade-off, and
is a big win for machines like the above.

The Problem:

The VM system is almost, but not quite prepared to cooperate
with a pmap module that wants vm_page_array to have holes.
I found that I need to make some refinements to the vm_page.c
function pmap_startup() before "holes" would work.  The
changes are backward-compatible with existing pmap code.

First, I'll provide an updated specification for the improved
interface between pmap_startup() and the pmap module.  That
is followed by the changes to vm_page.c for this interface.

(Comments appreciated)
Gordon Ross


Improved Interface:  (using MACHINE_NONCONTIG)

/* pmap.h */

/*
 * pmap_page_index()
 *
 * Given a physical address, return the page index.
 * The main advantage of the "hole" scheme used here is
 * that this frequently called function can be a macro!
 */

#define	pmap_page_index(pa) sun3_btop(pa)

/* pmap.c */

/*
 * For simplicity, this interface retains the variables
 * that were used in the old interface (without NONCONTIG).
 * These are set in pmap_bootstrap() and used in
 * pmap_next_page().
 */
/* Kernel virtual address space available: */
extern vm_offset_t virtual_avail, virtual_end;
/* Physical address space available: */
extern vm_offset_t avail_start, avail_end;
/* The "hole" (used to skip the Sun3/50 video RAM) */
extern vm_offset_t hole_start, hole_size;

/*
 * For our convenience, vm_page.c implements:
 *	 pmap_startup(), pmap_steal_memory()
 * using the functions:
 *	 pmap_virtual_space(), pmap_free_pages(), pmap_next_page(),
 * which are much simpler to implement.
 */

/*
 * How much virtual space does this kernel have?
 * (After mapping kernel text, data, etc.)
 */
void
pmap_virtual_space(v_start, v_end)
	vm_offset_t *v_start;
	vm_offset_t *v_end;
{
	*v_start = virtual_avail;
	*v_end   = virtual_end;
}


/*
 * Return the number of page indices in the range of
 * possible return values for pmap_page_index() for
 * all addresses provided by pmap_next_page().  This
 * return value is used to allocate per-page data.
 *
 * Note that a machine with a "hole" in physical memory
 * may include the pages in the hole in this count, and
 * skip the pages in the hole in pmap_next_page().
 */
u_int
pmap_free_pages()
{
	int bytes;

	bytes = avail_end - avail_start;
	return(sun3_btop(bytes));
}

/*
 * If there are still physical pages available, put the address of
 * the next available one at paddr and return non-zero, otherwise
 * return zero to indicate that there are no more free pages.
 * Note that avail_next is set to avail_start in _bootstrap().
 *
 * Imporant:  The page indices of the pages returned here must be
 * in ascending order.
 */
int
pmap_next_page(paddr)
	vm_offset_t *paddr;
{
	/* Is it time to skip over the hole? */
	if (avail_next == hole_start)
		avail_next += sun3_round_page(hole_size);

	/* Used up all ranges? */
	if (avail_next >= avail_end)
		return FALSE;

	/* Still have more memory... */
	*paddr = avail_next;
	avail_next += NBPG;

	return TRUE;
}

/*
 * pmap_bootstrap_alloc()
 * Note:  Not used with MACHINE_NONCONTIG
 */


Finally, here are the changes for vm/vm_page.c:

*** vm_page.c.orig	Mon Nov 21 17:49:39 1994
--- vm_page.c	Tue Nov 29 14:37:28 1994
***************
*** 492,530 ****
  	vm_offset_t	*startp;
  	vm_offset_t	*endp;
  {
! 	unsigned int	i;
  	vm_offset_t	paddr;
  	
  	/*
! 	 *	We calculate how many page frames we will have
! 	 *	and then allocate the page structures in one chunk.
  	 */
  	
! 	vm_page_count = ((PAGE_SIZE * pmap_free_pages() +
! 		   (round_page(virtual_space_start) - virtual_space_start)) /
! 		  (PAGE_SIZE + sizeof *vm_page_array));
! 	
! 	vm_page_array = (vm_page_t) pmap_steal_memory(vm_page_count
! 						      * sizeof *vm_page_array);
! 	
  	/*
! 	 *	Initialize the page frames.
  	 */
- 	
  	for (i = 0; i < vm_page_count; i++) {
! 		if (!pmap_next_page(&paddr))
! 			break;
! 		
  		VM_PAGE_INIT(&vm_page_array[i], NULL, NULL);
  		vm_page_array[i].phys_addr = paddr;
  		vm_page_free(&vm_page_array[i]);
  	}
! 	/*
! 	 *	Remember the actual page count and the index of the first page
! 	 */
! 	vm_page_count = i;
! 	first_page = pmap_page_index(vm_page_array[0].phys_addr);
! 	
  	*startp = virtual_space_start;
  	*endp = virtual_space_end;
  }
--- 492,564 ----
  	vm_offset_t	*startp;
  	vm_offset_t	*endp;
  {
! 	unsigned int	i, pmap_idx;
! 	unsigned int	freepages;
  	vm_offset_t	paddr;
  	
  	/*
! 	 * We calculate how many page frames we will have
! 	 * and then allocate the page structures in one chunk.
! 	 * The calculation is non-trivial.  We want:
! 	 *
! 	 *	vmpages > (freepages - (vmpages / sizeof(vm_page_t)))
! 	 *
! 	 * which, with some algebra, becomes:
! 	 *
! 	 *	vmpages > (freepages * sizeof(...) / (1 + sizeof(...)))
! 	 *
! 	 * The value of vm_page_count need not be exact, but must be
! 	 * large enough so vm_page_array handles the index range.
  	 */
+ 
+ 	freepages = pmap_free_pages();
+ 	/* Fudge slightly to deal with truncation error. */
+ 	freepages += 1;	/* fudge */
+ 
+ 	vm_page_count = (PAGE_SIZE * freepages) /
+ 		(PAGE_SIZE + sizeof(*vm_page_array));
+ 
+ 	vm_page_array = (vm_page_t)
+ 		pmap_steal_memory(vm_page_count * sizeof(*vm_page_array));
  	
! #ifdef	DIAGNOSTIC
  	/*
! 	 * Zero everyting in case the holes are used...
! 	 * XXX - Set to something that will cause a panic.
  	 */
  	for (i = 0; i < vm_page_count; i++) {
! 		bzero(&vm_page_array[i], sizeof(*vm_page_array));
! 	}
! #endif
! 
! 	/*
! 	 *	Initialize the page frames.
! 	 *	Note that some page indices may not be usable
! 	 *	when pmap_free_pages() counts pages in a hole.
! 	 *	Values returned by pmap_page_index() are the
! 	 *	array indices in vm_page_array directly.
! 	 */
! 	if (!pmap_next_page(&paddr))
! 		panic("pmap_startup: can't get first page");
! 	first_page = pmap_page_index(paddr);
! 	i = 0;
! 	for (;;) {
! 		/* Initialize a page array element. */
  		VM_PAGE_INIT(&vm_page_array[i], NULL, NULL);
  		vm_page_array[i].phys_addr = paddr;
  		vm_page_free(&vm_page_array[i]);
+ 
+ 		/* Are there more physical pages? */
+ 		if (!pmap_next_page(&paddr))
+ 			break;
+ 		pmap_idx = pmap_page_index(paddr);
+ 		i = pmap_idx - first_page;
+ 
+ 		/* Don't trust pmap_page_index()... */
+ 		if (i < 0 || i >= vm_page_count)
+ 			panic("pmap_startup: bad i=0x%x", i);
  	}
! 
  	*startp = virtual_space_start;
  	*endp = virtual_space_end;
  }