exa: create general m2mf rect copy for each chipset, use for UTS/DFS

No point having duplicated UTS/DFS logic in each implementation.  This also
splits nv50 out into its own code, in preparation for it being made to not
use relocs at all.

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

6 files changed, 327 insertions, 420 deletions

diff --git a/src/nouveau_exa.c b/src/nouveau_exa.c
index 824af70..e2f1496 100644
--- a/src/nouveau_exa.c
+++ b/src/nouveau_exa.c
@@ -43,231 +43,25 @@ NVAccelMemcpyRect(char *dst, const char *src, int height, int dst_pitch,
 	return TRUE;
 }
 
-static inline Bool
-NVAccelDownloadM2MF(PixmapPtr pspix, int x, int y, int w, int h,
-		    char *dst, unsigned dst_pitch)
-{
-	ScrnInfoPtr pScrn = xf86Screens[pspix->drawable.pScreen->myNum];
-	NVPtr pNv = NVPTR(pScrn);
-	struct nouveau_channel *chan = pNv->chan;
-	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
-	struct nouveau_bo *bo = nouveau_pixmap_bo(pspix);
-	unsigned cpp = pspix->drawable.bitsPerPixel / 8;
-	unsigned line_len = w * cpp;
-	unsigned src_offset = 0, src_pitch = 0, linear = 0;
-	/* Maximum DMA transfer */
-	unsigned line_count = pNv->GART->size / line_len;
-
-	if (!nv50_style_tiled_pixmap(pspix)) {
-		linear     = 1;
-		src_pitch  = exaGetPixmapPitch(pspix);
-		src_offset += (y * src_pitch) + (x * cpp);
-	}
-
-	/* HW limitations */
-	if (line_count > 2047)
-		line_count = 2047;
-
-	while (h) {
-		int i;
-		char *src;
-
-		if (line_count > h)
-			line_count = h;
-
-		if (MARK_RING(chan, 32, 6))
-			return FALSE;
-
-		BEGIN_RING(chan, m2mf, NV03_M2MF_DMA_BUFFER_IN, 2);
-		if (OUT_RELOCo(chan, bo, NOUVEAU_BO_GART | NOUVEAU_BO_VRAM |
-			       NOUVEAU_BO_RD) ||
-		    OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART |
-			       NOUVEAU_BO_WR)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-
-		if (pNv->Architecture >= NV_ARCH_50) {
-			if (!linear) {
-				BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_IN, 7);
-				OUT_RING  (chan, 0);
-				OUT_RING  (chan, bo->tile_mode << 4);
-				OUT_RING  (chan, pspix->drawable.width * cpp);
-				OUT_RING  (chan, pspix->drawable.height);
-				OUT_RING  (chan, 1);
-				OUT_RING  (chan, 0);
-				OUT_RING  (chan, (y << 16) | (x * cpp));
-			} else {
-				BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_IN, 1);
-				OUT_RING  (chan, 1);
-			}
-
-			BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_OUT, 1);
-			OUT_RING  (chan, 1);
-
-			BEGIN_RING(chan, m2mf, NV50_M2MF_OFFSET_IN_HIGH, 2);
-			if (OUT_RELOCh(chan, bo, src_offset, NOUVEAU_BO_GART |
-				       NOUVEAU_BO_VRAM | NOUVEAU_BO_RD) ||
-			    OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
-				       NOUVEAU_BO_WR)) {
-				MARK_UNDO(chan);
-				return FALSE;
-			}
-		}
-
-		BEGIN_RING(chan, m2mf, NV03_M2MF_OFFSET_IN, 8);
-		if (OUT_RELOCl(chan, bo, src_offset, NOUVEAU_BO_GART |
-			       NOUVEAU_BO_VRAM | NOUVEAU_BO_RD) ||
-		    OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART |
-			       NOUVEAU_BO_WR)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-		OUT_RING  (chan, src_pitch);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_count);
-		OUT_RING  (chan, (1<<8)|1);
-		OUT_RING  (chan, 0);
-
-		if (nouveau_bo_map(pNv->GART, NOUVEAU_BO_RD)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-		src = pNv->GART->map;
-		if (dst_pitch == line_len) {
-			memcpy(dst, src, dst_pitch * line_count);
-			dst += dst_pitch * line_count;
-		} else {
-			for (i = 0; i < line_count; i++) {
-				memcpy(dst, src, line_len);
-				src += line_len;
-				dst += dst_pitch;
-			}
-		}
-		nouveau_bo_unmap(pNv->GART);
-
-		if (linear)
-			src_offset += line_count * src_pitch;
-		h -= line_count;
-		y += line_count;
-	}
-
-	return TRUE;
-}
-
-static inline Bool
-NVAccelUploadM2MF(PixmapPtr pdpix, int x, int y, int w, int h,
-		  const char *src, int src_pitch)
+static Bool
+NVAccelM2MF(NVPtr pNv, int w, int h, int cpp,
+	    struct nouveau_bo *src, int sd, int sp, int sh, int sx, int sy,
+	    struct nouveau_bo *dst, int dd, int dp, int dh, int dx, int dy)
 {
-	ScrnInfoPtr pScrn = xf86Screens[pdpix->drawable.pScreen->myNum];
-	NVPtr pNv = NVPTR(pScrn);
-	struct nouveau_channel *chan = pNv->chan;
-	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
-	struct nouveau_bo *bo = nouveau_pixmap_bo(pdpix);
-	unsigned cpp = pdpix->drawable.bitsPerPixel / 8;
-	unsigned line_len = w * cpp;
-	unsigned dst_offset = 0, dst_pitch = 0, linear = 0;
-	/* Maximum DMA transfer */
-	unsigned line_count = pNv->GART->size / line_len;
-
-	if (!nv50_style_tiled_pixmap(pdpix)) {
-		linear     = 1;
-		dst_pitch  = exaGetPixmapPitch(pdpix);
-		dst_offset += (y * dst_pitch) + (x * cpp);
-	}
-
-	/* HW limitations */
-	if (line_count > 2047)
-		line_count = 2047;
-
-	while (h) {
-		int i;
-		char *dst;
-
-		if (line_count > h)
-			line_count = h;
-
-		/* Upload to GART */
-		if (nouveau_bo_map(pNv->GART, NOUVEAU_BO_WR))
-			return FALSE;
-		dst = pNv->GART->map;
-		if (src_pitch == line_len) {
-			memcpy(dst, src, src_pitch * line_count);
-			src += src_pitch * line_count;
-		} else {
-			for (i = 0; i < line_count; i++) {
-				memcpy(dst, src, line_len);
-				src += src_pitch;
-				dst += line_len;
-			}
-		}
-		nouveau_bo_unmap(pNv->GART);
-
-		if (MARK_RING(chan, 32, 6))
-			return FALSE;
-
-		BEGIN_RING(chan, m2mf, NV03_M2MF_DMA_BUFFER_IN, 2);
-		if (OUT_RELOCo(chan, pNv->GART, NOUVEAU_BO_GART |
-			       NOUVEAU_BO_RD) ||
-		    OUT_RELOCo(chan, bo, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART |
-			       NOUVEAU_BO_WR)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-
-		if (pNv->Architecture >= NV_ARCH_50) {
-			BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_IN, 1);
-			OUT_RING  (chan, 1);
-
-			if (!linear) {
-				BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_OUT, 7);
-				OUT_RING  (chan, 0);
-				OUT_RING  (chan, bo->tile_mode << 4);
-				OUT_RING  (chan, pdpix->drawable.width * cpp);
-				OUT_RING  (chan, pdpix->drawable.height);
-				OUT_RING  (chan, 1);
-				OUT_RING  (chan, 0);
-				OUT_RING  (chan, (y << 16) | (x * cpp));
-			} else {
-				BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_OUT, 1);
-				OUT_RING  (chan, 1);
-			}
-
-			BEGIN_RING(chan, m2mf, NV50_M2MF_OFFSET_IN_HIGH, 2);
-			if (OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO_GART |
-				       NOUVEAU_BO_RD) ||
-			    OUT_RELOCh(chan, bo, dst_offset, NOUVEAU_BO_VRAM |
-				       NOUVEAU_BO_GART | NOUVEAU_BO_WR)) {
-				MARK_UNDO(chan);
-				return FALSE;
-			}
-		}
-
-		/* DMA to VRAM */
-		BEGIN_RING(chan, m2mf, NV03_M2MF_OFFSET_IN, 8);
-		if (OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO_GART |
-			       NOUVEAU_BO_RD) ||
-		    OUT_RELOCl(chan, bo, dst_offset, NOUVEAU_BO_VRAM |
-			       NOUVEAU_BO_GART | NOUVEAU_BO_WR)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, dst_pitch);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_count);
-		OUT_RING  (chan, (1<<8)|1);
-		OUT_RING  (chan, 0);
-		FIRE_RING (chan);
-
-		if (linear)
-			dst_offset += line_count * dst_pitch;
-		h -= line_count;
-		y += line_count;
-	}
-
-	return TRUE;
+	if (pNv->Architecture >= NV_ARCH_C0)
+		return NVC0EXARectM2MF(pNv, w, h, cpp,
+				       src, sd, sp, sh, sx, sy,
+				       dst, dd, dp, dh, dx, dy);
+	else
+	if (pNv->Architecture >= NV_ARCH_50)
+		return NV50EXARectM2MF(pNv, w, h, cpp,
+				       src, sd, sp, sh, sx, sy,
+				       dst, dd, dp, dh, dx, dy);
+	else
+		return NV04EXARectM2MF(pNv, w, h, cpp,
+				       src, sd, sp, sh, sx, sy,
+				       dst, dd, dp, dh, dx, dy);
+	return FALSE;
 }
 
 static int
@@ -369,7 +163,8 @@ nouveau_exa_download_from_screen(PixmapPtr pspix, int x, int y, int w, int h,
 	ScrnInfoPtr pScrn = xf86Screens[pspix->drawable.pScreen->myNum];
 	NVPtr pNv = NVPTR(pScrn);
 	struct nouveau_bo *bo;
-	int src_pitch, cpp, offset;
+	int src_pitch, tmp_pitch, cpp, offset;
+	int max_lines, lines, i;
 	const char *src;
 	Bool ret;
 
@@ -377,18 +172,42 @@ nouveau_exa_download_from_screen(PixmapPtr pspix, int x, int y, int w, int h,
 	cpp = pspix->drawable.bitsPerPixel >> 3;
 	offset = (y * src_pitch) + (x * cpp);
 
-	if (pNv->GART) {
-		if (pNv->Architecture >= NV_ARCH_C0) {
-			if (NVC0AccelDownloadM2MF(pspix, x, y, w, h,
-						  dst, dst_pitch))
-				return TRUE;
+	if (!pNv->GART)
+		goto memcpy;
+
+	tmp_pitch = w * cpp;
+	max_lines = pNv->GART->size / tmp_pitch;
+	while (h) {
+		lines = max_lines;
+		if (lines > h)
+			lines = h;
+
+		if (!NVAccelM2MF(pNv, w, lines, cpp, nouveau_pixmap_bo(pspix),
+				 NOUVEAU_BO_VRAM, src_pitch,
+				 pspix->drawable.height, x, y, pNv->GART,
+				 NOUVEAU_BO_GART, tmp_pitch, lines, 0, 0))
+			goto memcpy;
+
+		nouveau_bo_map(pNv->GART, NOUVEAU_BO_RD);
+		if (src_pitch == tmp_pitch) {
+			memcpy(dst, pNv->GART->map, dst_pitch * lines);
+			dst += dst_pitch * lines;
 		} else {
-			if (NVAccelDownloadM2MF(pspix, x, y, w, h,
-						dst, dst_pitch))
-				return TRUE;
+			src = pNv->GART->map;
+			for (i = 0; i < lines; i++) {
+				memcpy(dst, src, tmp_pitch);
+				src += tmp_pitch;
+				dst += dst_pitch;
+			}
 		}
+		nouveau_bo_unmap(pNv->GART);
+
+		/* next! */
+		h -= lines;
+		y += lines;
 	}
 
+memcpy:
 	bo = nouveau_pixmap_bo(pspix);
 	if (nouveau_bo_map(bo, NOUVEAU_BO_RD))
 		return FALSE;
@@ -404,8 +223,9 @@ nouveau_exa_upload_to_screen(PixmapPtr pdpix, int x, int y, int w, int h,
 {
 	ScrnInfoPtr pScrn = xf86Screens[pdpix->drawable.pScreen->myNum];
 	NVPtr pNv = NVPTR(pScrn);
+	int dst_pitch, tmp_pitch, cpp;
+	int max_lines, lines, i;
 	struct nouveau_bo *bo;
-	int dst_pitch, cpp;
 	char *dst;
 	Bool ret;
 
@@ -438,22 +258,46 @@ nouveau_exa_upload_to_screen(PixmapPtr pdpix, int x, int y, int w, int h,
 	}
 
 	/* try gart-based transfer */
-	if (pNv->GART) {
-		if (pNv->Architecture < NV_ARCH_C0) {
-			ret = NVAccelUploadM2MF(pdpix, x, y, w, h,
-						src, src_pitch);
+	if (!pNv->GART)
+		goto memcpy;
+
+	tmp_pitch = w * cpp;
+	max_lines = pNv->GART->size / tmp_pitch;
+	while (h) {
+		lines = max_lines;
+		if (lines > h)
+			lines = h;
+
+		nouveau_bo_map(pNv->GART, NOUVEAU_BO_WR);
+		if (src_pitch == tmp_pitch) {
+			memcpy(pNv->GART->map, src, src_pitch * lines);
+			src += src_pitch * lines;
 		} else {
-			ret = NVC0AccelUploadM2MF(pdpix, x, y, w, h,
-						  src, src_pitch);
+			dst = pNv->GART->map;
+			for (i = 0; i < lines; i++) {
+				memcpy(dst, src, tmp_pitch);
+				src += src_pitch;
+				dst += tmp_pitch;
+			}
 		}
+		nouveau_bo_unmap(pNv->GART);
 
-		if (ret) {
-			exaMarkSync(pdpix->drawable.pScreen);
-			return TRUE;
-		}
+		if (!NVAccelM2MF(pNv, w, lines, cpp, pNv->GART,
+				 NOUVEAU_BO_GART, tmp_pitch, lines, 0, 0,
+				 nouveau_pixmap_bo(pdpix), NOUVEAU_BO_VRAM,
+				 dst_pitch, pdpix->drawable.height, x, y))
+			goto memcpy;
+
+		/* next! */
+		h -= lines;
+		y += lines;
 	}
 
+	exaMarkSync(pdpix->drawable.pScreen);
+	return TRUE;
+
 	/* fallback to memcpy-based transfer */
+memcpy:
 	bo = nouveau_pixmap_bo(pdpix);
 	if (nouveau_bo_map(bo, NOUVEAU_BO_WR))
 		return FALSE;
diff --git a/src/nv04_exa.c b/src/nv04_exa.c
index b27259b..4e12da2 100644
--- a/src/nv04_exa.c
+++ b/src/nv04_exa.c
@@ -25,6 +25,8 @@
 
 #include "nv04_pushbuf.h"
 
+#include "hwdefs/nv_object.xml.h"
+#include "hwdefs/nv_m2mf.xml.h"
 #include "hwdefs/nv01_2d.xml.h"
 
 static void 
@@ -421,4 +423,52 @@ NV04EXAUploadIFC(ScrnInfoPtr pScrn, const char *src, int src_pitch,
 	return TRUE;
 }
 
+Bool
+NV04EXARectM2MF(NVPtr pNv, int w, int h, int cpp,
+		struct nouveau_bo *src, int src_dom, int src_pitch,
+		int src_h, int src_x, int src_y, struct nouveau_bo *dst,
+		int dst_dom, int dst_pitch, int dst_h, int dst_x, int dst_y)
+{
+	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
+	struct nouveau_channel *chan = m2mf->channel;
+	unsigned src_off = src_y * src_pitch + src_x * cpp;
+	unsigned dst_off = dst_y * dst_pitch + dst_x * cpp;
+
+	while (h) {
+		int line_count = h;
+		if (line_count > 2047)
+			line_count = 2047;
+		h -= line_count;
+
+		if (MARK_RING (chan, 16, 4))
+			return FALSE;
 
+		BEGIN_RING(chan, m2mf, NV03_M2MF_DMA_BUFFER_IN, 2);
+		if (OUT_RELOCo(chan, src, src_dom | NOUVEAU_BO_RD) ||
+		    OUT_RELOCo(chan, dst, dst_dom | NOUVEAU_BO_WR)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+		BEGIN_RING(chan, m2mf, NV03_M2MF_OFFSET_IN, 8);
+		if (OUT_RELOCl(chan, src, src_off, src_dom | NOUVEAU_BO_RD) ||
+		    OUT_RELOCl(chan, dst, dst_off, dst_dom | NOUVEAU_BO_WR)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+		OUT_RING  (chan, src_pitch);
+		OUT_RING  (chan, dst_pitch);
+		OUT_RING  (chan, w * cpp);
+		OUT_RING  (chan, line_count);
+		OUT_RING  (chan, 0x00000101);
+		OUT_RING  (chan, 0x00000000);
+		BEGIN_RING(chan, m2mf, NV04_GRAPH_NOP, 1);
+		OUT_RING  (chan, 0x00000000);
+		BEGIN_RING(chan, m2mf, NV03_M2MF_OFFSET_OUT, 1);
+		OUT_RING  (chan, 0x00000000);
+
+		src_off += src_pitch * line_count;
+		dst_off += dst_pitch * line_count;
+	}
+
+	return TRUE;
+}
diff --git a/src/nv50_accel.h b/src/nv50_accel.h
index 7ee4f1f..01a99b8 100644
--- a/src/nv50_accel.h
+++ b/src/nv50_accel.h
@@ -8,6 +8,7 @@
 #include "hwdefs/nv50_defs.xml.h"
 #include "hwdefs/nv50_texture.h"
 #include "hwdefs/nv_3ddefs.xml.h"
+#include "hwdefs/nv_m2mf.xml.h"
 
 /* "Tesla scratch buffer" offsets */
 #define PVP_OFFSET  0x00000000 /* Vertex program */
diff --git a/src/nv50_exa.c b/src/nv50_exa.c
index 3719e89..a2f9b34 100644
--- a/src/nv50_exa.c
+++ b/src/nv50_exa.c
@@ -999,3 +999,92 @@ NV50EXADoneComposite(PixmapPtr pdpix)
 	chan->flush_notify = NULL;
 }
 
+Bool
+NV50EXARectM2MF(NVPtr pNv, int w, int h, int cpp,
+		struct nouveau_bo *src, int src_dom, int src_pitch,
+		int src_h, int src_x, int src_y, struct nouveau_bo *dst,
+		int dst_dom, int dst_pitch, int dst_h, int dst_x, int dst_y)
+{
+	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
+	struct nouveau_channel *chan = m2mf->channel;
+	unsigned src_off = 0, dst_off = 0;
+
+	if (src->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+		BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_IN, 6);
+		OUT_RING  (chan, 0);
+		OUT_RING  (chan, src->tile_mode << 4);
+		OUT_RING  (chan, src_pitch);
+		OUT_RING  (chan, src_h);
+		OUT_RING  (chan, 1);
+		OUT_RING  (chan, 0);
+	} else {
+		BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_IN, 1);
+		OUT_RING  (chan, 1);
+		BEGIN_RING(chan, m2mf, NV03_M2MF_PITCH_IN, 1);
+		OUT_RING  (chan, src_pitch);
+	}
+
+	if (dst->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+		BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_OUT, 6);
+		OUT_RING  (chan, 0);
+		OUT_RING  (chan, dst->tile_mode << 4);
+		OUT_RING  (chan, dst_pitch);
+		OUT_RING  (chan, dst_h);
+		OUT_RING  (chan, 1);
+		OUT_RING  (chan, 0);
+	} else {
+		BEGIN_RING(chan, m2mf, NV50_M2MF_LINEAR_OUT, 1);
+		OUT_RING  (chan, 1);
+		BEGIN_RING(chan, m2mf, NV03_M2MF_PITCH_OUT, 1);
+		OUT_RING  (chan, dst_pitch);
+	}
+
+	while (h) {
+		int line_count = h;
+		if (line_count > 2047)
+			line_count = 2047;
+
+		if (MARK_RING (chan, 32, 4))
+			return FALSE;
+
+		if (src->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+			BEGIN_RING(chan, m2mf, NV50_M2MF_TILING_POSITION_IN, 1);
+			OUT_RING  (chan, (src_y << 16) | (src_x * cpp));
+		} else {
+			src_off = src_y * src_pitch + src_x * cpp;
+		}
+
+		if (dst->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+			BEGIN_RING(chan, m2mf, NV50_M2MF_TILING_POSITION_OUT, 1);
+			OUT_RING  (chan, (dst_y << 16) | (dst_x * cpp));
+		} else {
+			dst_off = dst_y * dst_pitch + dst_x * cpp;
+		}
+
+		BEGIN_RING(chan, m2mf, NV50_M2MF_OFFSET_IN_HIGH, 2);
+		if (OUT_RELOCh(chan, src, src_off, src_dom | NOUVEAU_BO_RD) ||
+		    OUT_RELOCh(chan, dst, dst_off, dst_dom | NOUVEAU_BO_WR)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+
+		BEGIN_RING(chan, m2mf, NV03_M2MF_OFFSET_IN, 2);
+		if (OUT_RELOCl(chan, src, src_off, src_dom | NOUVEAU_BO_RD) ||
+		    OUT_RELOCl(chan, dst, dst_off, dst_dom | NOUVEAU_BO_WR)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+
+		BEGIN_RING(chan, m2mf, NV03_M2MF_LINE_LENGTH_IN, 4);
+		OUT_RING  (chan, w * cpp);
+		OUT_RING  (chan, line_count);
+		OUT_RING  (chan, 0x00000101);
+		OUT_RING  (chan, 0x00000000);
+
+		src_y += line_count;
+		dst_y += line_count;
+		h  -= line_count;
+	}
+
+	return TRUE;
+}
diff --git a/src/nv_proto.h b/src/nv_proto.h
index 2e4fce0..67093e1 100644
--- a/src/nv_proto.h
+++ b/src/nv_proto.h
@@ -78,6 +78,9 @@ void NV04EXACopy(PixmapPtr, int, int, int, int, int, int);
 void NV04EXADoneCopy(PixmapPtr);
 Bool NV04EXAUploadIFC(ScrnInfoPtr, const char *src, int src_pitch,
 		      PixmapPtr pdPix, int x, int y, int w, int h, int cpp);
+Bool NV04EXARectM2MF(NVPtr pNv, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int);
 
 /* in nv10_exa.c */
 Bool NVAccelInitNV10TCL(ScrnInfoPtr pScrn);
@@ -153,6 +156,9 @@ void NV50EXAComposite(PixmapPtr, int, int, int, int, int, int, int, int);
 void NV50EXADoneComposite(PixmapPtr);
 Bool NV50EXAUploadSIFC(const char *src, int src_pitch,
 		       PixmapPtr pdPix, int x, int y, int w, int h, int cpp);
+Bool NV50EXARectM2MF(NVPtr pNv, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int);
 
 /* in nvc0_exa.c */
 Bool NVC0AccelUploadM2MF(PixmapPtr pdpix, int x, int y, int w, int h,
@@ -173,6 +179,9 @@ void NVC0EXAComposite(PixmapPtr, int, int, int, int, int, int, int, int);
 void NVC0EXADoneComposite(PixmapPtr);
 Bool NVC0EXAUploadSIFC(const char *src, int src_pitch,
 		       PixmapPtr pdPix, int x, int y, int w, int h, int cpp);
+Bool NVC0EXARectM2MF(NVPtr pNv, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int,
+		     struct nouveau_bo *, int, int, int, int, int);
 
 /* nv50_xv.c */
 int nv50_xv_image_put(ScrnInfoPtr, struct nouveau_bo *, int, int, int, int,
diff --git a/src/nvc0_exa.c b/src/nvc0_exa.c
index def0e5f..c553e68 100644
--- a/src/nvc0_exa.c
+++ b/src/nvc0_exa.c
@@ -27,182 +27,6 @@
 
 #define NOUVEAU_BO(a, b, c) (NOUVEAU_BO_##a | NOUVEAU_BO_##b | NOUVEAU_BO_##c)
 
-Bool
-NVC0AccelDownloadM2MF(PixmapPtr pspix, int x, int y, int w, int h,
-		      char *dst, unsigned dst_pitch)
-{
-	ScrnInfoPtr pScrn = xf86Screens[pspix->drawable.pScreen->myNum];
-	NVPtr pNv = NVPTR(pScrn);
-	struct nouveau_channel *chan = pNv->chan;
-	struct nouveau_bo *bo = nouveau_pixmap_bo(pspix);
-	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
-	const int cpp = pspix->drawable.bitsPerPixel / 8;
-	const int line_len = w * cpp;
-	const int line_limit = (128 << 10) / line_len;
-	unsigned src_offset = 0, src_pitch = 0, tiled = 1;
-
-	if (!nv50_style_tiled_pixmap(pspix)) {
-		tiled = 0;
-		src_pitch = exaGetPixmapPitch(pspix);
-		src_offset = (y * src_pitch) + (x * cpp);
-	} else {
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_MODE_IN, 5);
-		OUT_RING  (chan, bo->tile_mode);
-		OUT_RING  (chan, pspix->drawable.width * cpp);
-		OUT_RING  (chan, pspix->drawable.height);
-		OUT_RING  (chan, 1);
-		OUT_RING  (chan, 0);
-	}
-
-	while (h) {
-		const char *src;
-		int line_count, i;
-
-		/* GART size >= 128 KiB assumed */
-		line_count = h;
-		if (line_count > line_limit)
-			line_count = line_limit;
-
-		MARK_RING(chan, 16, 4);
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_OUT_HIGH, 2);
-		OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO(GART, GART, WR));
-		OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO(GART, GART, WR));
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_IN_HIGH, 6);
-		OUT_RELOCh(chan, bo, src_offset, NOUVEAU_BO(VRAM, GART, RD));
-		OUT_RELOCl(chan, bo, src_offset, NOUVEAU_BO(VRAM, GART, RD));
-		OUT_RING  (chan, src_pitch);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_count);
-
-		if (tiled) {
-			BEGIN_RING(chan, m2mf,
-				   NVC0_M2MF_TILING_POSITION_IN_X, 2);
-			OUT_RING  (chan, x * cpp);
-			OUT_RING  (chan, y);
-		}
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_EXEC, 1);
-		OUT_RING  (chan, 0x100000 | (tiled << 8));
-
-		if (nouveau_bo_map(pNv->GART, NOUVEAU_BO_RD)) {
-			MARK_UNDO(chan);
-			return FALSE;
-		}
-		src = pNv->GART->map;
-
-		if (dst_pitch == line_len) {
-			memcpy(dst, src, dst_pitch * line_count);
-			dst += dst_pitch * line_count;
-		} else {
-			for (i = 0; i < line_count; ++i) {
-				memcpy(dst, src, line_len);
-				src += line_len;
-				dst += dst_pitch;
-			}
-		}
-		nouveau_bo_unmap(pNv->GART);
-
-		if (!tiled)
-			src_offset += line_count * src_pitch;
-		h -= line_count;
-		y += line_count;
-	}
-
-	return TRUE;
-}
-
-Bool
-NVC0AccelUploadM2MF(PixmapPtr pdpix, int x, int y, int w, int h,
-		    const char *src, int src_pitch)
-{
-	ScrnInfoPtr pScrn = xf86Screens[pdpix->drawable.pScreen->myNum];
-	NVPtr pNv = NVPTR(pScrn);
-	struct nouveau_channel *chan = pNv->chan;
-	struct nouveau_bo *bo = nouveau_pixmap_bo(pdpix);
-	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
-	int cpp = pdpix->drawable.bitsPerPixel / 8;
-	int line_len = w * cpp;
-	int line_limit = (128 << 10) / line_len;
-	unsigned dst_offset = 0, dst_pitch = 0, tiled = 1;
-
-	if (!nv50_style_tiled_pixmap(pdpix)) {
-		tiled = 0;
-		dst_pitch = exaGetPixmapPitch(pdpix);
-		dst_offset = (y * dst_pitch) + (x * cpp);
-	} else {
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_MODE_OUT, 5);
-		OUT_RING  (chan, bo->tile_mode);
-		OUT_RING  (chan, pdpix->drawable.width * cpp);
-		OUT_RING  (chan, pdpix->drawable.height);
-		OUT_RING  (chan, 1);
-		OUT_RING  (chan, 0);
-	}
-
-	while (h) {
-		char *dst;
-		int i, line_count;
-
-		line_count = h;
-		if (line_count > line_limit)
-			line_count = line_limit;
-
-		if (nouveau_bo_map(pNv->GART, NOUVEAU_BO_WR))
-			return FALSE;
-		dst = pNv->GART->map;
-
-		if (src_pitch == line_len) {
-			memcpy(dst, src, src_pitch * line_count);
-			src += src_pitch * line_count;
-		} else {
-			for (i = 0; i < line_count; i++) {
-				memcpy(dst, src, line_len);
-				src += src_pitch;
-				dst += line_len;
-                        }
-		}
-		nouveau_bo_unmap(pNv->GART);
-
-		if (MARK_RING(chan, 16, 4))
-			return FALSE;
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_IN_HIGH, 2);
-		OUT_RELOCh(chan, pNv->GART, 0, NOUVEAU_BO(GART, GART, RD));
-		OUT_RELOCl(chan, pNv->GART, 0, NOUVEAU_BO(GART, GART, RD));
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_OUT_HIGH, 2);
-		OUT_RELOCh(chan, bo, dst_offset, NOUVEAU_BO(VRAM, GART, WR));
-		OUT_RELOCl(chan, bo, dst_offset, NOUVEAU_BO(VRAM, GART, WR));
-
-		if (tiled) {
-			BEGIN_RING(chan, m2mf,
-				   NVC0_M2MF_TILING_POSITION_OUT_X, 2);
-			OUT_RING  (chan, x * cpp);
-			OUT_RING  (chan, y);
-		}
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_PITCH_IN, 4);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, dst_pitch);
-		OUT_RING  (chan, line_len);
-		OUT_RING  (chan, line_count);
-
-		BEGIN_RING(chan, m2mf, NVC0_M2MF_EXEC, 1);
-		OUT_RING  (chan, 0x100000 | (tiled << 4));
-		FIRE_RING (chan);
-
-		if (!tiled)
-			dst_offset += line_count * dst_pitch;
-		h -= line_count;
-		y += line_count;
-	}
-
-	return TRUE;
-}
-
-
 struct nvc0_exa_state {
 	struct {
 		PictTransformPtr transform;
@@ -1201,3 +1025,93 @@ NVC0EXADoneComposite(PixmapPtr pdpix)
 	chan->flush_notify = NULL;
 }
 
+Bool
+NVC0EXARectM2MF(NVPtr pNv, int w, int h, int cpp,
+		struct nouveau_bo *src, int src_dom, int src_pitch,
+		int src_h, int src_x, int src_y, struct nouveau_bo *dst,
+		int dst_dom, int dst_pitch, int dst_h, int dst_x, int dst_y)
+{
+	struct nouveau_grobj *m2mf = pNv->NvMemFormat;
+	struct nouveau_channel *chan = m2mf->channel;
+	unsigned src_off = 0, dst_off = 0;
+	unsigned exec = 0;
+
+	if (src->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_MODE_IN, 5);
+		OUT_RING  (chan, src->tile_mode);
+		OUT_RING  (chan, src_pitch);
+		OUT_RING  (chan, src_h);
+		OUT_RING  (chan, 1);
+		OUT_RING  (chan, 0);
+	} else {
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_PITCH_IN, 1);
+		OUT_RING  (chan, src_pitch);
+
+		exec |= NVC0_M2MF_EXEC_LINEAR_IN;
+	}
+
+	if (dst->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_MODE_OUT, 5);
+		OUT_RING  (chan, dst->tile_mode);
+		OUT_RING  (chan, dst_pitch);
+		OUT_RING  (chan, dst_h);
+		OUT_RING  (chan, 1);
+		OUT_RING  (chan, 0);
+	} else {
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_PITCH_OUT, 1);
+		OUT_RING  (chan, dst_pitch);
+
+		exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
+	}
+
+	while (h) {
+		int line_count = h;
+		if (line_count > 2047)
+			line_count = 2047;
+
+		if (MARK_RING (chan, 32, 4))
+			return FALSE;
+
+		if (src->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+			BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_POSITION_IN_X, 2);
+			OUT_RING  (chan, src_x * cpp);
+			OUT_RING  (chan, src_y);
+		} else {
+			src_off = src_y * src_pitch + src_x * cpp;
+		}
+
+		if (dst->tile_flags & NOUVEAU_BO_TILE_LAYOUT_MASK) {
+			BEGIN_RING(chan, m2mf, NVC0_M2MF_TILING_POSITION_OUT_X, 2);
+			OUT_RING  (chan, dst_x * cpp);
+			OUT_RING  (chan, dst_y);
+		} else {
+			dst_off = dst_y * dst_pitch + dst_x * cpp;
+		}
+
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_OUT_HIGH, 2);
+		if (OUT_RELOCh(chan, dst, dst_off, dst_dom | NOUVEAU_BO_WR) ||
+		    OUT_RELOCl(chan, dst, dst_off, dst_dom | NOUVEAU_BO_WR)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_OFFSET_IN_HIGH, 2);
+		if (OUT_RELOCh(chan, src, src_off, src_dom | NOUVEAU_BO_RD) ||
+		    OUT_RELOCl(chan, src, src_off, src_dom | NOUVEAU_BO_RD)) {
+			MARK_UNDO(chan);
+			return FALSE;
+		}
+
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_LINE_LENGTH_IN, 2);
+		OUT_RING  (chan, w * cpp);
+		OUT_RING  (chan, line_count);
+		BEGIN_RING(chan, m2mf, NVC0_M2MF_EXEC, 1);
+		OUT_RING  (chan, NVC0_M2MF_EXEC_QUERY_SHORT | exec);
+
+		src_y += line_count;
+		dst_y += line_count;
+		h  -= line_count;
+	}
+
+	return TRUE;
+}