Add PCI DMA memory functions and make addbufs_pci and associated code use

    it. To do this we need to save the bus address along with the virtual
    address in the seglist. Also fix some error handling and a few bits of
    whitespace.

12 files changed, 380 insertions, 160 deletions

diff --git a/bsd-core/drmP.h b/bsd-core/drmP.h
index d40bcdaa..e8720395 100644
--- a/bsd-core/drmP.h
+++ b/bsd-core/drmP.h
@@ -196,7 +196,8 @@ typedef struct drm_buf_entry {
 	drm_buf_t	  *buflist;
 	int		  seg_count;
 	int		  page_order;
-	unsigned long	  *seglist;
+	vm_offset_t	  *seglist;
+	dma_addr_t	  *seglist_bus;
 
 	drm_freelist_t	  freelist;
 } drm_buf_entry_t;
@@ -562,6 +563,12 @@ extern int		DRM(sg_alloc)(DRM_IOCTL_ARGS);
 extern int		DRM(sg_free)(DRM_IOCTL_ARGS);
 #endif
 
+/* consistent PCI memory functions (drm_pci.h) */
+extern void		*DRM(pci_alloc)(drm_device_t *dev, size_t size, 
+					size_t align, dma_addr_t maxaddr,
+					dma_addr_t *busaddr);
+extern void		DRM(pci_free)(drm_device_t *dev, size_t size, 
+				      void *vaddr, dma_addr_t busaddr);
 
 #endif /* __KERNEL__ */
 #endif /* _DRM_P_H_ */
diff --git a/bsd-core/drm_bufs.c b/bsd-core/drm_bufs.c
index f792140e..87ec6f09 100644
--- a/bsd-core/drm_bufs.c
+++ b/bsd-core/drm_bufs.c
@@ -141,7 +141,7 @@ int DRM(addmap)( DRM_IOCTL_ARGS )
 
 	case _DRM_SHM:
 		map->handle = (void *)DRM(alloc)(map->size, DRM_MEM_SAREA);
-		DRM_DEBUG( "%ld %d %p\n",
+		DRM_DEBUG( "%lu %d %p\n",
 			   map->size, DRM(order)( map->size ), map->handle );
 		if ( !map->handle ) {
 			DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
@@ -279,31 +279,33 @@ int DRM(rmmap)( DRM_IOCTL_ARGS )
 #if __HAVE_DMA
 
 
-static void DRM(cleanup_buf_error)(drm_buf_entry_t *entry)
+static void DRM(cleanup_buf_error)(drm_device_t *dev, drm_buf_entry_t *entry)
 {
 	int i;
 
+#if __HAVE_PCI_DMA
 	if (entry->seg_count) {
 		for (i = 0; i < entry->seg_count; i++) {
-			DRM(free)((void *)entry->seglist[i],
-					entry->buf_size,
-					DRM_MEM_DMA);
+			if (entry->seglist[i] != NULL)
+				DRM(pci_free)(dev, entry->buf_size,
+				    (void *)entry->seglist[i],
+				    entry->seglist_bus[i]);
 		}
 		DRM(free)(entry->seglist,
 			  entry->seg_count *
 			  sizeof(*entry->seglist),
 			  DRM_MEM_SEGS);
+		DRM(free)(entry->seglist_bus, entry->seg_count *
+			  sizeof(*entry->seglist_bus), DRM_MEM_SEGS);
 
 		entry->seg_count = 0;
 	}
+#endif /* __HAVE_PCI_DMA */
 
-   	if(entry->buf_count) {
-	   	for(i = 0; i < entry->buf_count; i++) {
-			if(entry->buflist[i].dev_private) {
-				DRM(free)(entry->buflist[i].dev_private,
-					  entry->buflist[i].dev_priv_size,
-					  DRM_MEM_BUFS);
-			}
+   	if (entry->buf_count) {
+	   	for (i = 0; i < entry->buf_count; i++) {
+			DRM(free)(entry->buflist[i].dev_private,
+			    entry->buflist[i].dev_priv_size, DRM_MEM_BUFS);
 		}
 		DRM(free)(entry->buflist,
 			  entry->buf_count *
@@ -395,7 +397,9 @@ static int DRM(addbufs_agp)(drm_device_t *dev, drm_buf_desc_t *request)
 		if(!buf->dev_private) {
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
-			DRM(cleanup_buf_error)(entry);
+			DRM(cleanup_buf_error)(dev, entry);
+			DRM_UNLOCK;
+			return DRM_ERR(ENOMEM);
 		}
 		memset( buf->dev_private, 0, buf->dev_priv_size );
 
@@ -413,7 +417,7 @@ static int DRM(addbufs_agp)(drm_device_t *dev, drm_buf_desc_t *request)
 				     DRM_MEM_BUFS );
 	if(!temp_buflist) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
@@ -450,7 +454,7 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	int total;
 	int page_order;
 	drm_buf_entry_t *entry;
-	unsigned long page;
+	vm_offset_t vaddr;
 	drm_buf_t *buf;
 	int alignment;
 	unsigned long offset;
@@ -459,6 +463,7 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	int page_count;
 	unsigned long *temp_pagelist;
 	drm_buf_t **temp_buflist;
+	dma_addr_t bus_addr;
 
 	count = request->count;
 	order = DRM(order)(request->size);
@@ -482,42 +487,37 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 		return DRM_ERR(ENOMEM);	/* May only call once for each order */
 	}
 
-	entry->buflist = DRM(alloc)( count * sizeof(*entry->buflist),
-				    DRM_MEM_BUFS );
-	if ( !entry->buflist ) {
-		DRM_UNLOCK;
-		return DRM_ERR(ENOMEM);
-	}
-	memset( entry->buflist, 0, count * sizeof(*entry->buflist) );
+	entry->buflist = DRM(alloc)(count * sizeof(*entry->buflist),
+	    DRM_MEM_BUFS);
+	entry->seglist = DRM(alloc)(count * sizeof(*entry->seglist),
+	    DRM_MEM_SEGS);
+	entry->seglist_bus = DRM(alloc)(count * sizeof(*entry->seglist_bus),
+	    DRM_MEM_SEGS);
 
-	entry->seglist = DRM(alloc)( count * sizeof(*entry->seglist),
-				    DRM_MEM_SEGS );
-	if ( !entry->seglist ) {
-		DRM(free)( entry->buflist,
-			  count * sizeof(*entry->buflist),
-			  DRM_MEM_BUFS );
-		DRM_UNLOCK;
-		return DRM_ERR(ENOMEM);
-	}
-	memset( entry->seglist, 0, count * sizeof(*entry->seglist) );
-
-	temp_pagelist = DRM(realloc)( dma->pagelist,
-				      dma->page_count * sizeof(*dma->pagelist),
-				      (dma->page_count + (count << page_order))
-				      * sizeof(*dma->pagelist),
-				      DRM_MEM_PAGES );
-	if(!temp_pagelist) {
-		DRM(free)( entry->buflist,
-			   count * sizeof(*entry->buflist),
-			   DRM_MEM_BUFS );
-		DRM(free)( entry->seglist,
-			   count * sizeof(*entry->seglist),
-			   DRM_MEM_SEGS );
+	/* Keep the original pagelist until we know all the allocations
+	 * have succeeded
+	 */
+	temp_pagelist = DRM(alloc)((dma->page_count + (count << page_order)) *
+	    sizeof(*dma->pagelist), DRM_MEM_PAGES);
+
+	if (entry->buflist == NULL || entry->seglist == NULL || 
+	    temp_pagelist == NULL) {
+		DRM(free)(entry->buflist, count * sizeof(*entry->buflist),
+		    DRM_MEM_BUFS);
+		DRM(free)(entry->seglist, count * sizeof(*entry->seglist),
+		    DRM_MEM_SEGS);
+		DRM(free)(entry->seglist_bus, count *
+		    sizeof(*entry->seglist_bus), DRM_MEM_SEGS);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
 
-	dma->pagelist = temp_pagelist;
+	bzero(entry->buflist, count * sizeof(*entry->buflist));
+	bzero(entry->seglist, count * sizeof(*entry->seglist));
+	
+	memcpy(temp_pagelist, dma->pagelist, dma->page_count * 
+	    sizeof(*dma->pagelist));
+
 	DRM_DEBUG( "pagelist: %d entries\n",
 		   dma->page_count + (count << page_order) );
 
@@ -527,15 +527,28 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	page_count = 0;
 
 	while ( entry->buf_count < count ) {
-		page = (unsigned long)DRM(alloc)( size, DRM_MEM_DMA );
-		if ( !page ) break;
-		entry->seglist[entry->seg_count++] = page;
+		vaddr = (vm_offset_t) DRM(pci_alloc)(dev, size, alignment,
+		    0xfffffffful, &bus_addr);
+		if (vaddr == NULL) {
+			/* Set count correctly so we free the proper amount. */
+			entry->buf_count = count;
+			entry->seg_count = count;
+			DRM(cleanup_buf_error)(dev, entry);
+			DRM(free)(temp_pagelist, (dma->page_count +
+			    (count << page_order)) * sizeof(*dma->pagelist),
+			    DRM_MEM_PAGES);
+			DRM_UNLOCK;
+			return DRM_ERR(ENOMEM);
+		}
+	
+		entry->seglist_bus[entry->seg_count] = bus_addr;
+		entry->seglist[entry->seg_count++] = vaddr;
 		for ( i = 0 ; i < (1 << page_order) ; i++ ) {
 			DRM_DEBUG( "page %d @ 0x%08lx\n",
 				   dma->page_count + page_count,
 				   page + PAGE_SIZE * i );
-			dma->pagelist[dma->page_count + page_count++]
-				= page + PAGE_SIZE * i;
+			temp_pagelist[dma->page_count + page_count++] = 
+			    vaddr + PAGE_SIZE * i;
 		}
 		for ( offset = 0 ;
 		      offset + size <= total && entry->buf_count < count ;
@@ -546,10 +559,28 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 			buf->order   = order;
 			buf->used    = 0;
 			buf->offset  = (dma->byte_count + byte_count + offset);
-			buf->address = (void *)(page + offset);
+			buf->address = (void *)(vaddr + offset);
+			buf->bus_address = bus_addr + offset;
 			buf->next    = NULL;
 			buf->pending = 0;
 			buf->filp    = NULL;
+
+			buf->dev_priv_size = sizeof(DRIVER_BUF_PRIV_T);
+			buf->dev_private = DRM(alloc)(sizeof(DRIVER_BUF_PRIV_T),
+			    DRM_MEM_BUFS);
+			if (buf->dev_private == NULL) {
+				/* Set count correctly so we free the proper amount. */
+				entry->buf_count = count;
+				entry->seg_count = count;
+				DRM(cleanup_buf_error)(dev, entry);
+				DRM(free)(temp_pagelist, (dma->page_count + 
+				    (count << page_order)) *
+				    sizeof(*dma->pagelist), DRM_MEM_PAGES );
+				DRM_UNLOCK;
+				return DRM_ERR(ENOMEM);
+			}
+			bzero(buf->dev_private, buf->dev_priv_size);
+
 			DRM_DEBUG( "buffer %d @ %p\n",
 				   entry->buf_count, buf->address );
 		}
@@ -561,9 +592,12 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 				     (dma->buf_count + entry->buf_count)
 				     * sizeof(*dma->buflist),
 				     DRM_MEM_BUFS );
-	if(!temp_buflist) {
+	if (temp_buflist == NULL) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
+		DRM(free)(temp_pagelist, (dma->page_count + 
+		    (count << page_order)) * sizeof(*dma->pagelist),
+		    DRM_MEM_PAGES);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
@@ -573,6 +607,13 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 		dma->buflist[i + dma->buf_count] = &entry->buflist[i];
 	}
 
+	/* No allocations failed, so now we can replace the orginal pagelist
+	 * with the new one.
+	 */
+	DRM(free)(dma->pagelist, dma->page_count * sizeof(*dma->pagelist),
+	    DRM_MEM_PAGES);
+	dma->pagelist = temp_pagelist;
+
 	dma->buf_count += entry->buf_count;
 	dma->seg_count += entry->seg_count;
 	dma->page_count += entry->seg_count << page_order;
@@ -669,7 +710,7 @@ static int DRM(addbufs_sg)(drm_device_t *dev, drm_buf_desc_t *request)
 		if(!buf->dev_private) {
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
-			DRM(cleanup_buf_error)(entry);
+			DRM(cleanup_buf_error)(dev, entry);
 			DRM_UNLOCK;
 			return DRM_ERR(ENOMEM);
 		}
@@ -693,7 +734,7 @@ static int DRM(addbufs_sg)(drm_device_t *dev, drm_buf_desc_t *request)
 				     DRM_MEM_BUFS );
 	if(!temp_buflist) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
diff --git a/bsd-core/drm_dma.c b/bsd-core/drm_dma.c
index 4ddf8b22..526faee2 100644
--- a/bsd-core/drm_dma.c
+++ b/bsd-core/drm_dma.c
@@ -70,6 +70,7 @@ void DRM(dma_takedown)(drm_device_t *dev)
 
 				/* Clear dma buffers */
 	for (i = 0; i <= DRM_MAX_ORDER; i++) {
+#if __HAVE_PCI_DMA
 		if (dma->bufs[i].seg_count) {
 			DRM_DEBUG("order %d: buf_count = %d,"
 				  " seg_count = %d\n",
@@ -77,22 +78,27 @@ void DRM(dma_takedown)(drm_device_t *dev)
 				  dma->bufs[i].buf_count,
 				  dma->bufs[i].seg_count);
 			for (j = 0; j < dma->bufs[i].seg_count; j++) {
-				DRM(free)((void *)dma->bufs[i].seglist[j],
-						dma->bufs[i].buf_size,
-						DRM_MEM_DMA);
+				if (dma->bufs[i].seglist[j] != NULL)
+					DRM(pci_free)(dev, dma->bufs[i].buf_size,
+					    (void *)dma->bufs[i].seglist[j],
+					    dma->bufs[i].seglist_bus[j]);
 			}
 			DRM(free)(dma->bufs[i].seglist,
 				  dma->bufs[i].seg_count
 				  * sizeof(*dma->bufs[0].seglist),
 				  DRM_MEM_SEGS);
+			DRM(free)(dma->bufs[i].seglist_bus,
+				  dma->bufs[i].seg_count
+				  * sizeof(*dma->bufs[0].seglist_bus),
+				  DRM_MEM_SEGS);
 		}
-	   	if(dma->bufs[i].buf_count) {
-		   	for(j = 0; j < dma->bufs[i].buf_count; j++) {
-			   if(dma->bufs[i].buflist[j].dev_private) {
-			      DRM(free)(dma->bufs[i].buflist[j].dev_private,
+#endif /* __HAVE_PCI_DMA */
+
+	   	if (dma->bufs[i].buf_count) {
+		   	for (j = 0; j < dma->bufs[i].buf_count; j++) {
+				DRM(free)(dma->bufs[i].buflist[j].dev_private,
 					dma->bufs[i].buflist[j].dev_priv_size,
 					DRM_MEM_BUFS);
-			   }
 			}
 		   	DRM(free)(dma->bufs[i].buflist,
 				  dma->bufs[i].buf_count *
@@ -101,17 +107,10 @@ void DRM(dma_takedown)(drm_device_t *dev)
 		}
 	}
 
-	if (dma->buflist) {
-		DRM(free)(dma->buflist,
-			  dma->buf_count * sizeof(*dma->buflist),
-			  DRM_MEM_BUFS);
-	}
-
-	if (dma->pagelist) {
-		DRM(free)(dma->pagelist,
-			  dma->page_count * sizeof(*dma->pagelist),
-			  DRM_MEM_PAGES);
-	}
+	DRM(free)(dma->buflist, dma->buf_count * sizeof(*dma->buflist),
+	    DRM_MEM_BUFS);
+	DRM(free)(dma->pagelist, dma->page_count * sizeof(*dma->pagelist),
+	    DRM_MEM_PAGES);
 	DRM(free)(dev->dma, sizeof(*dev->dma), DRM_MEM_DRIVER);
 	dev->dma = NULL;
 }
diff --git a/bsd-core/drm_pci.c b/bsd-core/drm_pci.c
new file mode 100644
index 00000000..aa065b64
--- /dev/null
+++ b/bsd-core/drm_pci.c
@@ -0,0 +1,65 @@
+/**
+ * \file drm_pci.h
+ * \brief PCI consistent, DMA-accessible memory functions.
+ *
+ * \author Eric Anholt <anholt@FreeBSD.org>
+ */
+
+/*
+ * Copyright 2003 Eric Anholt.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "drmP.h"
+
+/**********************************************************************/
+/** \name PCI memory */
+/*@{*/
+
+/**
+ * \brief Allocate a physically contiguous DMA-accessible consistent 
+ * memory block.
+ */
+void *
+DRM(pci_alloc)(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr,
+    dma_addr_t *busaddr)
+{
+	void *vaddr;
+
+	vaddr = contigmalloc(size, DRM(M_DRM), M_WAITOK, 0ul, maxaddr, align,
+	    0);
+	*busaddr = vtophys(vaddr);
+	
+	return vaddr;
+}
+
+/**
+ * \brief Free a DMA-accessible consistent memory block.
+ */
+void
+DRM(pci_free)(drm_device_t *dev, size_t size, void *vaddr, dma_addr_t busaddr)
+{
+#if __FreeBSD_version > 500000
+	contigfree(vaddr, size, DRM(M_DRM));	/* Not available on 4.x */
+#endif
+}
+
+/*@}*/
diff --git a/bsd-core/r128_drv.c b/bsd-core/r128_drv.c
index e7d18959..0aae76e8 100644
--- a/bsd-core/r128_drv.c
+++ b/bsd-core/r128_drv.c
@@ -75,11 +75,10 @@ drm_chipinfo_t DRM(devicelist)[] = {
 #include "drm_ioctl.h"
 #include "drm_lock.h"
 #include "drm_memory.h"
+#include "drm_pci.h"
 #include "drm_sysctl.h"
 #include "drm_vm.h"
-#if __HAVE_SG
 #include "drm_scatter.h"
-#endif
 
 #ifdef __FreeBSD__
 DRIVER_MODULE(r128, pci, r128_driver, r128_devclass, 0, 0);
diff --git a/bsd-core/radeon_drv.c b/bsd-core/radeon_drv.c
index badd7081..5ad5d4b0 100644
--- a/bsd-core/radeon_drv.c
+++ b/bsd-core/radeon_drv.c
@@ -88,11 +88,10 @@ drm_chipinfo_t DRM(devicelist)[] = {
 #include "drm_ioctl.h"
 #include "drm_lock.h"
 #include "drm_memory.h"
+#include "drm_pci.h"
 #include "drm_vm.h"
 #include "drm_sysctl.h"
-#if __HAVE_SG
 #include "drm_scatter.h"
-#endif
 
 #ifdef __FreeBSD__
 DRIVER_MODULE(DRIVER_NAME, pci, DRM(driver), DRM(devclass), 0, 0);
diff --git a/bsd/drmP.h b/bsd/drmP.h
index d40bcdaa..e8720395 100644
--- a/bsd/drmP.h
+++ b/bsd/drmP.h
@@ -196,7 +196,8 @@ typedef struct drm_buf_entry {
 	drm_buf_t	  *buflist;
 	int		  seg_count;
 	int		  page_order;
-	unsigned long	  *seglist;
+	vm_offset_t	  *seglist;
+	dma_addr_t	  *seglist_bus;
 
 	drm_freelist_t	  freelist;
 } drm_buf_entry_t;
@@ -562,6 +563,12 @@ extern int		DRM(sg_alloc)(DRM_IOCTL_ARGS);
 extern int		DRM(sg_free)(DRM_IOCTL_ARGS);
 #endif
 
+/* consistent PCI memory functions (drm_pci.h) */
+extern void		*DRM(pci_alloc)(drm_device_t *dev, size_t size, 
+					size_t align, dma_addr_t maxaddr,
+					dma_addr_t *busaddr);
+extern void		DRM(pci_free)(drm_device_t *dev, size_t size, 
+				      void *vaddr, dma_addr_t busaddr);
 
 #endif /* __KERNEL__ */
 #endif /* _DRM_P_H_ */
diff --git a/bsd/drm_bufs.h b/bsd/drm_bufs.h
index f792140e..87ec6f09 100644
--- a/bsd/drm_bufs.h
+++ b/bsd/drm_bufs.h
@@ -141,7 +141,7 @@ int DRM(addmap)( DRM_IOCTL_ARGS )
 
 	case _DRM_SHM:
 		map->handle = (void *)DRM(alloc)(map->size, DRM_MEM_SAREA);
-		DRM_DEBUG( "%ld %d %p\n",
+		DRM_DEBUG( "%lu %d %p\n",
 			   map->size, DRM(order)( map->size ), map->handle );
 		if ( !map->handle ) {
 			DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
@@ -279,31 +279,33 @@ int DRM(rmmap)( DRM_IOCTL_ARGS )
 #if __HAVE_DMA
 
 
-static void DRM(cleanup_buf_error)(drm_buf_entry_t *entry)
+static void DRM(cleanup_buf_error)(drm_device_t *dev, drm_buf_entry_t *entry)
 {
 	int i;
 
+#if __HAVE_PCI_DMA
 	if (entry->seg_count) {
 		for (i = 0; i < entry->seg_count; i++) {
-			DRM(free)((void *)entry->seglist[i],
-					entry->buf_size,
-					DRM_MEM_DMA);
+			if (entry->seglist[i] != NULL)
+				DRM(pci_free)(dev, entry->buf_size,
+				    (void *)entry->seglist[i],
+				    entry->seglist_bus[i]);
 		}
 		DRM(free)(entry->seglist,
 			  entry->seg_count *
 			  sizeof(*entry->seglist),
 			  DRM_MEM_SEGS);
+		DRM(free)(entry->seglist_bus, entry->seg_count *
+			  sizeof(*entry->seglist_bus), DRM_MEM_SEGS);
 
 		entry->seg_count = 0;
 	}
+#endif /* __HAVE_PCI_DMA */
 
-   	if(entry->buf_count) {
-	   	for(i = 0; i < entry->buf_count; i++) {
-			if(entry->buflist[i].dev_private) {
-				DRM(free)(entry->buflist[i].dev_private,
-					  entry->buflist[i].dev_priv_size,
-					  DRM_MEM_BUFS);
-			}
+   	if (entry->buf_count) {
+	   	for (i = 0; i < entry->buf_count; i++) {
+			DRM(free)(entry->buflist[i].dev_private,
+			    entry->buflist[i].dev_priv_size, DRM_MEM_BUFS);
 		}
 		DRM(free)(entry->buflist,
 			  entry->buf_count *
@@ -395,7 +397,9 @@ static int DRM(addbufs_agp)(drm_device_t *dev, drm_buf_desc_t *request)
 		if(!buf->dev_private) {
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
-			DRM(cleanup_buf_error)(entry);
+			DRM(cleanup_buf_error)(dev, entry);
+			DRM_UNLOCK;
+			return DRM_ERR(ENOMEM);
 		}
 		memset( buf->dev_private, 0, buf->dev_priv_size );
 
@@ -413,7 +417,7 @@ static int DRM(addbufs_agp)(drm_device_t *dev, drm_buf_desc_t *request)
 				     DRM_MEM_BUFS );
 	if(!temp_buflist) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
@@ -450,7 +454,7 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	int total;
 	int page_order;
 	drm_buf_entry_t *entry;
-	unsigned long page;
+	vm_offset_t vaddr;
 	drm_buf_t *buf;
 	int alignment;
 	unsigned long offset;
@@ -459,6 +463,7 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	int page_count;
 	unsigned long *temp_pagelist;
 	drm_buf_t **temp_buflist;
+	dma_addr_t bus_addr;
 
 	count = request->count;
 	order = DRM(order)(request->size);
@@ -482,42 +487,37 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 		return DRM_ERR(ENOMEM);	/* May only call once for each order */
 	}
 
-	entry->buflist = DRM(alloc)( count * sizeof(*entry->buflist),
-				    DRM_MEM_BUFS );
-	if ( !entry->buflist ) {
-		DRM_UNLOCK;
-		return DRM_ERR(ENOMEM);
-	}
-	memset( entry->buflist, 0, count * sizeof(*entry->buflist) );
+	entry->buflist = DRM(alloc)(count * sizeof(*entry->buflist),
+	    DRM_MEM_BUFS);
+	entry->seglist = DRM(alloc)(count * sizeof(*entry->seglist),
+	    DRM_MEM_SEGS);
+	entry->seglist_bus = DRM(alloc)(count * sizeof(*entry->seglist_bus),
+	    DRM_MEM_SEGS);
 
-	entry->seglist = DRM(alloc)( count * sizeof(*entry->seglist),
-				    DRM_MEM_SEGS );
-	if ( !entry->seglist ) {
-		DRM(free)( entry->buflist,
-			  count * sizeof(*entry->buflist),
-			  DRM_MEM_BUFS );
-		DRM_UNLOCK;
-		return DRM_ERR(ENOMEM);
-	}
-	memset( entry->seglist, 0, count * sizeof(*entry->seglist) );
-
-	temp_pagelist = DRM(realloc)( dma->pagelist,
-				      dma->page_count * sizeof(*dma->pagelist),
-				      (dma->page_count + (count << page_order))
-				      * sizeof(*dma->pagelist),
-				      DRM_MEM_PAGES );
-	if(!temp_pagelist) {
-		DRM(free)( entry->buflist,
-			   count * sizeof(*entry->buflist),
-			   DRM_MEM_BUFS );
-		DRM(free)( entry->seglist,
-			   count * sizeof(*entry->seglist),
-			   DRM_MEM_SEGS );
+	/* Keep the original pagelist until we know all the allocations
+	 * have succeeded
+	 */
+	temp_pagelist = DRM(alloc)((dma->page_count + (count << page_order)) *
+	    sizeof(*dma->pagelist), DRM_MEM_PAGES);
+
+	if (entry->buflist == NULL || entry->seglist == NULL || 
+	    temp_pagelist == NULL) {
+		DRM(free)(entry->buflist, count * sizeof(*entry->buflist),
+		    DRM_MEM_BUFS);
+		DRM(free)(entry->seglist, count * sizeof(*entry->seglist),
+		    DRM_MEM_SEGS);
+		DRM(free)(entry->seglist_bus, count *
+		    sizeof(*entry->seglist_bus), DRM_MEM_SEGS);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
 
-	dma->pagelist = temp_pagelist;
+	bzero(entry->buflist, count * sizeof(*entry->buflist));
+	bzero(entry->seglist, count * sizeof(*entry->seglist));
+	
+	memcpy(temp_pagelist, dma->pagelist, dma->page_count * 
+	    sizeof(*dma->pagelist));
+
 	DRM_DEBUG( "pagelist: %d entries\n",
 		   dma->page_count + (count << page_order) );
 
@@ -527,15 +527,28 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 	page_count = 0;
 
 	while ( entry->buf_count < count ) {
-		page = (unsigned long)DRM(alloc)( size, DRM_MEM_DMA );
-		if ( !page ) break;
-		entry->seglist[entry->seg_count++] = page;
+		vaddr = (vm_offset_t) DRM(pci_alloc)(dev, size, alignment,
+		    0xfffffffful, &bus_addr);
+		if (vaddr == NULL) {
+			/* Set count correctly so we free the proper amount. */
+			entry->buf_count = count;
+			entry->seg_count = count;
+			DRM(cleanup_buf_error)(dev, entry);
+			DRM(free)(temp_pagelist, (dma->page_count +
+			    (count << page_order)) * sizeof(*dma->pagelist),
+			    DRM_MEM_PAGES);
+			DRM_UNLOCK;
+			return DRM_ERR(ENOMEM);
+		}
+	
+		entry->seglist_bus[entry->seg_count] = bus_addr;
+		entry->seglist[entry->seg_count++] = vaddr;
 		for ( i = 0 ; i < (1 << page_order) ; i++ ) {
 			DRM_DEBUG( "page %d @ 0x%08lx\n",
 				   dma->page_count + page_count,
 				   page + PAGE_SIZE * i );
-			dma->pagelist[dma->page_count + page_count++]
-				= page + PAGE_SIZE * i;
+			temp_pagelist[dma->page_count + page_count++] = 
+			    vaddr + PAGE_SIZE * i;
 		}
 		for ( offset = 0 ;
 		      offset + size <= total && entry->buf_count < count ;
@@ -546,10 +559,28 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 			buf->order   = order;
 			buf->used    = 0;
 			buf->offset  = (dma->byte_count + byte_count + offset);
-			buf->address = (void *)(page + offset);
+			buf->address = (void *)(vaddr + offset);
+			buf->bus_address = bus_addr + offset;
 			buf->next    = NULL;
 			buf->pending = 0;
 			buf->filp    = NULL;
+
+			buf->dev_priv_size = sizeof(DRIVER_BUF_PRIV_T);
+			buf->dev_private = DRM(alloc)(sizeof(DRIVER_BUF_PRIV_T),
+			    DRM_MEM_BUFS);
+			if (buf->dev_private == NULL) {
+				/* Set count correctly so we free the proper amount. */
+				entry->buf_count = count;
+				entry->seg_count = count;
+				DRM(cleanup_buf_error)(dev, entry);
+				DRM(free)(temp_pagelist, (dma->page_count + 
+				    (count << page_order)) *
+				    sizeof(*dma->pagelist), DRM_MEM_PAGES );
+				DRM_UNLOCK;
+				return DRM_ERR(ENOMEM);
+			}
+			bzero(buf->dev_private, buf->dev_priv_size);
+
 			DRM_DEBUG( "buffer %d @ %p\n",
 				   entry->buf_count, buf->address );
 		}
@@ -561,9 +592,12 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 				     (dma->buf_count + entry->buf_count)
 				     * sizeof(*dma->buflist),
 				     DRM_MEM_BUFS );
-	if(!temp_buflist) {
+	if (temp_buflist == NULL) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
+		DRM(free)(temp_pagelist, (dma->page_count + 
+		    (count << page_order)) * sizeof(*dma->pagelist),
+		    DRM_MEM_PAGES);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
@@ -573,6 +607,13 @@ static int DRM(addbufs_pci)(drm_device_t *dev, drm_buf_desc_t *request)
 		dma->buflist[i + dma->buf_count] = &entry->buflist[i];
 	}
 
+	/* No allocations failed, so now we can replace the orginal pagelist
+	 * with the new one.
+	 */
+	DRM(free)(dma->pagelist, dma->page_count * sizeof(*dma->pagelist),
+	    DRM_MEM_PAGES);
+	dma->pagelist = temp_pagelist;
+
 	dma->buf_count += entry->buf_count;
 	dma->seg_count += entry->seg_count;
 	dma->page_count += entry->seg_count << page_order;
@@ -669,7 +710,7 @@ static int DRM(addbufs_sg)(drm_device_t *dev, drm_buf_desc_t *request)
 		if(!buf->dev_private) {
 			/* Set count correctly so we free the proper amount. */
 			entry->buf_count = count;
-			DRM(cleanup_buf_error)(entry);
+			DRM(cleanup_buf_error)(dev, entry);
 			DRM_UNLOCK;
 			return DRM_ERR(ENOMEM);
 		}
@@ -693,7 +734,7 @@ static int DRM(addbufs_sg)(drm_device_t *dev, drm_buf_desc_t *request)
 				     DRM_MEM_BUFS );
 	if(!temp_buflist) {
 		/* Free the entry because it isn't valid */
-		DRM(cleanup_buf_error)(entry);
+		DRM(cleanup_buf_error)(dev, entry);
 		DRM_UNLOCK;
 		return DRM_ERR(ENOMEM);
 	}
diff --git a/bsd/drm_dma.h b/bsd/drm_dma.h
index 4ddf8b22..526faee2 100644
--- a/bsd/drm_dma.h
+++ b/bsd/drm_dma.h
@@ -70,6 +70,7 @@ void DRM(dma_takedown)(drm_device_t *dev)
 
 				/* Clear dma buffers */
 	for (i = 0; i <= DRM_MAX_ORDER; i++) {
+#if __HAVE_PCI_DMA
 		if (dma->bufs[i].seg_count) {
 			DRM_DEBUG("order %d: buf_count = %d,"
 				  " seg_count = %d\n",
@@ -77,22 +78,27 @@ void DRM(dma_takedown)(drm_device_t *dev)
 				  dma->bufs[i].buf_count,
 				  dma->bufs[i].seg_count);
 			for (j = 0; j < dma->bufs[i].seg_count; j++) {
-				DRM(free)((void *)dma->bufs[i].seglist[j],
-						dma->bufs[i].buf_size,
-						DRM_MEM_DMA);
+				if (dma->bufs[i].seglist[j] != NULL)
+					DRM(pci_free)(dev, dma->bufs[i].buf_size,
+					    (void *)dma->bufs[i].seglist[j],
+					    dma->bufs[i].seglist_bus[j]);
 			}
 			DRM(free)(dma->bufs[i].seglist,
 				  dma->bufs[i].seg_count
 				  * sizeof(*dma->bufs[0].seglist),
 				  DRM_MEM_SEGS);
+			DRM(free)(dma->bufs[i].seglist_bus,
+				  dma->bufs[i].seg_count
+				  * sizeof(*dma->bufs[0].seglist_bus),
+				  DRM_MEM_SEGS);
 		}
-	   	if(dma->bufs[i].buf_count) {
-		   	for(j = 0; j < dma->bufs[i].buf_count; j++) {
-			   if(dma->bufs[i].buflist[j].dev_private) {
-			      DRM(free)(dma->bufs[i].buflist[j].dev_private,
+#endif /* __HAVE_PCI_DMA */
+
+	   	if (dma->bufs[i].buf_count) {
+		   	for (j = 0; j < dma->bufs[i].buf_count; j++) {
+				DRM(free)(dma->bufs[i].buflist[j].dev_private,
 					dma->bufs[i].buflist[j].dev_priv_size,
 					DRM_MEM_BUFS);
-			   }
 			}
 		   	DRM(free)(dma->bufs[i].buflist,
 				  dma->bufs[i].buf_count *
@@ -101,17 +107,10 @@ void DRM(dma_takedown)(drm_device_t *dev)
 		}
 	}
 
-	if (dma->buflist) {
-		DRM(free)(dma->buflist,
-			  dma->buf_count * sizeof(*dma->buflist),
-			  DRM_MEM_BUFS);
-	}
-
-	if (dma->pagelist) {
-		DRM(free)(dma->pagelist,
-			  dma->page_count * sizeof(*dma->pagelist),
-			  DRM_MEM_PAGES);
-	}
+	DRM(free)(dma->buflist, dma->buf_count * sizeof(*dma->buflist),
+	    DRM_MEM_BUFS);
+	DRM(free)(dma->pagelist, dma->page_count * sizeof(*dma->pagelist),
+	    DRM_MEM_PAGES);
 	DRM(free)(dev->dma, sizeof(*dev->dma), DRM_MEM_DRIVER);
 	dev->dma = NULL;
 }
diff --git a/bsd/drm_pci.h b/bsd/drm_pci.h
new file mode 100644
index 00000000..aa065b64
--- /dev/null
+++ b/bsd/drm_pci.h
@@ -0,0 +1,65 @@
+/**
+ * \file drm_pci.h
+ * \brief PCI consistent, DMA-accessible memory functions.
+ *
+ * \author Eric Anholt <anholt@FreeBSD.org>
+ */
+
+/*
+ * Copyright 2003 Eric Anholt.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "drmP.h"
+
+/**********************************************************************/
+/** \name PCI memory */
+/*@{*/
+
+/**
+ * \brief Allocate a physically contiguous DMA-accessible consistent 
+ * memory block.
+ */
+void *
+DRM(pci_alloc)(drm_device_t *dev, size_t size, size_t align, dma_addr_t maxaddr,
+    dma_addr_t *busaddr)
+{
+	void *vaddr;
+
+	vaddr = contigmalloc(size, DRM(M_DRM), M_WAITOK, 0ul, maxaddr, align,
+	    0);
+	*busaddr = vtophys(vaddr);
+	
+	return vaddr;
+}
+
+/**
+ * \brief Free a DMA-accessible consistent memory block.
+ */
+void
+DRM(pci_free)(drm_device_t *dev, size_t size, void *vaddr, dma_addr_t busaddr)
+{
+#if __FreeBSD_version > 500000
+	contigfree(vaddr, size, DRM(M_DRM));	/* Not available on 4.x */
+#endif
+}
+
+/*@}*/
diff --git a/bsd/r128_drv.c b/bsd/r128_drv.c
index e7d18959..0aae76e8 100644
--- a/bsd/r128_drv.c
+++ b/bsd/r128_drv.c
@@ -75,11 +75,10 @@ drm_chipinfo_t DRM(devicelist)[] = {
 #include "drm_ioctl.h"
 #include "drm_lock.h"
 #include "drm_memory.h"
+#include "drm_pci.h"
 #include "drm_sysctl.h"
 #include "drm_vm.h"
-#if __HAVE_SG
 #include "drm_scatter.h"
-#endif
 
 #ifdef __FreeBSD__
 DRIVER_MODULE(r128, pci, r128_driver, r128_devclass, 0, 0);
diff --git a/bsd/radeon_drv.c b/bsd/radeon_drv.c
index badd7081..5ad5d4b0 100644
--- a/bsd/radeon_drv.c
+++ b/bsd/radeon_drv.c
@@ -88,11 +88,10 @@ drm_chipinfo_t DRM(devicelist)[] = {
 #include "drm_ioctl.h"
 #include "drm_lock.h"
 #include "drm_memory.h"
+#include "drm_pci.h"
 #include "drm_vm.h"
 #include "drm_sysctl.h"
-#if __HAVE_SG
 #include "drm_scatter.h"
-#endif
 
 #ifdef __FreeBSD__
 DRIVER_MODULE(DRIVER_NAME, pci, DRM(driver), DRM(devclass), 0, 0);