drm/nouveau/fifo: pre-move some blocks of code around

- will make subsequent patches more obvious
- no code changes

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

6 files changed, 400 insertions, 400 deletions

diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index 06e47728e3a4..b06db3b3d471 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -212,6 +212,14 @@ nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend)
 }
 
 static int
+nvkm_fifo_init(struct nvkm_engine *engine)
+{
+	struct nvkm_fifo *fifo = nvkm_fifo(engine);
+	fifo->func->init(fifo);
+	return 0;
+}
+
+static int
 nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data)
 {
 	struct nvkm_fifo *fifo = nvkm_fifo(engine);
@@ -240,14 +248,6 @@ nvkm_fifo_preinit(struct nvkm_engine *engine)
 	nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0);
 }
 
-static int
-nvkm_fifo_init(struct nvkm_engine *engine)
-{
-	struct nvkm_fifo *fifo = nvkm_fifo(engine);
-	fifo->func->init(fifo);
-	return 0;
-}
-
 static void *
 nvkm_fifo_dtor(struct nvkm_engine *engine)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
index d17d8c65d169..ed2ae9e69987 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c
@@ -69,72 +69,6 @@ nv04_fifo_dma_object_ctor(struct nvkm_fifo_chan *base,
 	return hash;
 }
 
-void
-nv04_fifo_dma_fini(struct nvkm_fifo_chan *base)
-{
-	struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
-	struct nv04_fifo *fifo = chan->fifo;
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	struct nvkm_memory *fctx = device->imem->ramfc;
-	const struct nv04_fifo_ramfc *c;
-	unsigned long flags;
-	u32 mask = fifo->base.nr - 1;
-	u32 data = chan->ramfc;
-	u32 chid;
-
-	/* prevent fifo context switches */
-	spin_lock_irqsave(&fifo->base.lock, flags);
-	nvkm_wr32(device, NV03_PFIFO_CACHES, 0);
-
-	/* if this channel is active, replace it with a null context */
-	chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & mask;
-	if (chid == chan->base.chid) {
-		nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
-		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0);
-		nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
-
-		c = fifo->ramfc;
-		nvkm_kmap(fctx);
-		do {
-			u32 rm = ((1ULL << c->bits) - 1) << c->regs;
-			u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
-			u32 rv = (nvkm_rd32(device, c->regp) &  rm) >> c->regs;
-			u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm);
-			nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
-		} while ((++c)->bits);
-		nvkm_done(fctx);
-
-		c = fifo->ramfc;
-		do {
-			nvkm_wr32(device, c->regp, 0x00000000);
-		} while ((++c)->bits);
-
-		nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0);
-		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0);
-		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, mask);
-		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1);
-		nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);
-	}
-
-	/* restore normal operation, after disabling dma mode */
-	nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0);
-	nvkm_wr32(device, NV03_PFIFO_CACHES, 1);
-	spin_unlock_irqrestore(&fifo->base.lock, flags);
-}
-
-void
-nv04_fifo_dma_init(struct nvkm_fifo_chan *base)
-{
-	struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
-	struct nv04_fifo *fifo = chan->fifo;
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	u32 mask = 1 << chan->base.chid;
-	unsigned long flags;
-	spin_lock_irqsave(&fifo->base.lock, flags);
-	nvkm_mask(device, NV04_PFIFO_MODE, mask, mask);
-	spin_unlock_irqrestore(&fifo->base.lock, flags);
-}
-
 void *
 nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 25eda4fd2867..bd0b0c66e28f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -53,6 +53,52 @@ static const struct nvkm_chan_func
 gf100_chan = {
 };
 
+static const struct nvkm_bitfield
+gf100_fifo_pbdma_intr[] = {
+/*	{ 0x00008000, "" }	seen with null ib push */
+	{ 0x00200000, "ILLEGAL_MTHD" },
+	{ 0x00800000, "EMPTY_SUBC" },
+	{}
+};
+
+static void
+gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit)
+{
+	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000));
+	u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000));
+	u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000));
+	u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0x7f;
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00003ffc);
+	struct nvkm_fifo_chan *chan;
+	unsigned long flags;
+	u32 show = stat;
+	char msg[128];
+
+	if (stat & 0x00800000) {
+		if (device->sw) {
+			if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data))
+				show &= ~0x00800000;
+		}
+	}
+
+	if (show) {
+		nvkm_snprintbf(msg, sizeof(msg), gf100_fifo_pbdma_intr, show);
+		chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags);
+		nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] "
+				   "subc %d mthd %04x data %08x\n",
+			   unit, show, msg, chid, chan ? chan->inst->addr : 0,
+			   chan ? chan->object.client->name : "unknown",
+			   subc, mthd, data);
+		nvkm_fifo_chan_put(&fifo->base, flags, &chan);
+	}
+
+	nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008);
+	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
+}
+
 void
 gf100_fifo_runlist_commit(struct gf100_fifo *fifo)
 {
@@ -399,52 +445,6 @@ gf100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit)
 	nvkm_fifo_fault(fifo, &info);
 }
 
-static const struct nvkm_bitfield
-gf100_fifo_pbdma_intr[] = {
-/*	{ 0x00008000, "" }	seen with null ib push */
-	{ 0x00200000, "ILLEGAL_MTHD" },
-	{ 0x00800000, "EMPTY_SUBC" },
-	{}
-};
-
-static void
-gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit)
-{
-	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
-	struct nvkm_device *device = subdev->device;
-	u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000));
-	u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000));
-	u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000));
-	u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0x7f;
-	u32 subc = (addr & 0x00070000) >> 16;
-	u32 mthd = (addr & 0x00003ffc);
-	struct nvkm_fifo_chan *chan;
-	unsigned long flags;
-	u32 show= stat;
-	char msg[128];
-
-	if (stat & 0x00800000) {
-		if (device->sw) {
-			if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data))
-				show &= ~0x00800000;
-		}
-	}
-
-	if (show) {
-		nvkm_snprintbf(msg, sizeof(msg), gf100_fifo_pbdma_intr, show);
-		chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags);
-		nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] "
-				   "subc %d mthd %04x data %08x\n",
-			   unit, show, msg, chid, chan ? chan->inst->addr : 0,
-			   chan ? chan->object.client->name : "unknown",
-			   subc, mthd, data);
-		nvkm_fifo_chan_put(&fifo->base, flags, &chan);
-	}
-
-	nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008);
-	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
-}
-
 static void
 gf100_fifo_intr_runlist(struct gf100_fifo *fifo)
 {
@@ -576,46 +576,6 @@ gf100_fifo_intr(struct nvkm_fifo *base)
 	}
 }
 
-static int
-gf100_fifo_oneinit(struct nvkm_fifo *base)
-{
-	struct gf100_fifo *fifo = gf100_fifo(base);
-	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
-	struct nvkm_device *device = subdev->device;
-	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
-	int ret;
-
-	/* Determine number of PBDMAs by checking valid enable bits. */
-	nvkm_wr32(device, 0x002204, 0xffffffff);
-	fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204));
-	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
-
-
-	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
-			      false, &fifo->runlist.mem[0]);
-	if (ret)
-		return ret;
-
-	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
-			      false, &fifo->runlist.mem[1]);
-	if (ret)
-		return ret;
-
-	init_waitqueue_head(&fifo->runlist.wait);
-
-	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000,
-			      0x1000, false, &fifo->user.mem);
-	if (ret)
-		return ret;
-
-	ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem),
-			   &fifo->user.bar);
-	if (ret)
-		return ret;
-
-	return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0);
-}
-
 static void
 gf100_fifo_fini(struct nvkm_fifo *base)
 {
@@ -659,6 +619,46 @@ gf100_fifo_init(struct nvkm_fifo *base)
 	nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
 }
 
+static int
+gf100_fifo_oneinit(struct nvkm_fifo *base)
+{
+	struct gf100_fifo *fifo = gf100_fifo(base);
+	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device);
+	int ret;
+
+	/* Determine number of PBDMAs by checking valid enable bits. */
+	nvkm_wr32(device, 0x002204, 0xffffffff);
+	fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204));
+	nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr);
+
+
+	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
+			      false, &fifo->runlist.mem[0]);
+	if (ret)
+		return ret;
+
+	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000,
+			      false, &fifo->runlist.mem[1]);
+	if (ret)
+		return ret;
+
+	init_waitqueue_head(&fifo->runlist.wait);
+
+	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000,
+			      0x1000, false, &fifo->user.mem);
+	if (ret)
+		return ret;
+
+	ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem),
+			   &fifo->user.bar);
+	if (ret)
+		return ret;
+
+	return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0);
+}
+
 static void *
 gf100_fifo_dtor(struct nvkm_fifo *base)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 1e15b88d59ae..da26c5219e53 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -103,6 +103,70 @@ gk104_fifo_uevent_init(struct nvkm_fifo *fifo)
 	nvkm_mask(device, 0x002140, 0x80000000, 0x80000000);
 }
 
+static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = {
+	{ 0x00000001, "HCE_RE_ILLEGAL_OP" },
+	{ 0x00000002, "HCE_RE_ALIGNB" },
+	{ 0x00000004, "HCE_PRIV" },
+	{ 0x00000008, "HCE_ILLEGAL_MTHD" },
+	{ 0x00000010, "HCE_ILLEGAL_CLASS" },
+	{}
+};
+
+void
+gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit)
+{
+	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000));
+	u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask;
+	u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff;
+	char msg[128];
+
+	if (stat) {
+		nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat);
+		nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n",
+			   unit, stat, msg, chid,
+			   nvkm_rd32(device, 0x040150 + (unit * 0x2000)),
+			   nvkm_rd32(device, 0x040154 + (unit * 0x2000)));
+	}
+
+	nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat);
+}
+
+static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
+	{ 0x00000001, "MEMREQ" },
+	{ 0x00000002, "MEMACK_TIMEOUT" },
+	{ 0x00000004, "MEMACK_EXTRA" },
+	{ 0x00000008, "MEMDAT_TIMEOUT" },
+	{ 0x00000010, "MEMDAT_EXTRA" },
+	{ 0x00000020, "MEMFLUSH" },
+	{ 0x00000040, "MEMOP" },
+	{ 0x00000080, "LBCONNECT" },
+	{ 0x00000100, "LBREQ" },
+	{ 0x00000200, "LBACK_TIMEOUT" },
+	{ 0x00000400, "LBACK_EXTRA" },
+	{ 0x00000800, "LBDAT_TIMEOUT" },
+	{ 0x00001000, "LBDAT_EXTRA" },
+	{ 0x00002000, "GPFIFO" },
+	{ 0x00004000, "GPPTR" },
+	{ 0x00008000, "GPENTRY" },
+	{ 0x00010000, "GPCRC" },
+	{ 0x00020000, "PBPTR" },
+	{ 0x00040000, "PBENTRY" },
+	{ 0x00080000, "PBCRC" },
+	{ 0x00100000, "XBARCONNECT" },
+	{ 0x00200000, "METHOD" },
+	{ 0x00400000, "METHODCRC" },
+	{ 0x00800000, "DEVICE" },
+	{ 0x02000000, "SEMAPHORE" },
+	{ 0x04000000, "ACQUIRE" },
+	{ 0x08000000, "PRI" },
+	{ 0x20000000, "NO_CTXSW_SEG" },
+	{ 0x40000000, "PBSEG" },
+	{ 0x80000000, "SIGNATURE" },
+	{}
+};
+
 void
 gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl,
 			  struct nvkm_memory *mem, int nr)
@@ -443,6 +507,128 @@ gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn)
 	schedule_work(&fifo->recover.work);
 }
 
+const struct nvkm_enum
+gk104_fifo_fault_access[] = {
+	{ 0x0, "READ" },
+	{ 0x1, "WRITE" },
+	{}
+};
+
+const struct nvkm_enum
+gk104_fifo_fault_engine[] = {
+	{ 0x00, "GR", NULL, NVKM_ENGINE_GR },
+	{ 0x01, "DISPLAY" },
+	{ 0x02, "CAPTURE" },
+	{ 0x03, "IFB", NULL, NVKM_ENGINE_IFB },
+	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
+	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
+	{ 0x06, "SCHED" },
+	{ 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO },
+	{ 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO },
+	{ 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0f, "HOSTSR" },
+	{ 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD },
+	{ 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP },
+	{ 0x13, "PERF" },
+	{ 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC },
+	{ 0x15, "CE0", NULL, NVKM_ENGINE_CE, 0 },
+	{ 0x16, "CE1", NULL, NVKM_ENGINE_CE, 1 },
+	{ 0x17, "PMU" },
+	{ 0x18, "PTP" },
+	{ 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC },
+	{ 0x1b, "CE2", NULL, NVKM_ENGINE_CE, 2 },
+	{}
+};
+
+const struct nvkm_enum
+gk104_fifo_fault_reason[] = {
+	{ 0x00, "PDE" },
+	{ 0x01, "PDE_SIZE" },
+	{ 0x02, "PTE" },
+	{ 0x03, "VA_LIMIT_VIOLATION" },
+	{ 0x04, "UNBOUND_INST_BLOCK" },
+	{ 0x05, "PRIV_VIOLATION" },
+	{ 0x06, "RO_VIOLATION" },
+	{ 0x07, "WO_VIOLATION" },
+	{ 0x08, "PITCH_MASK_VIOLATION" },
+	{ 0x09, "WORK_CREATION" },
+	{ 0x0a, "UNSUPPORTED_APERTURE" },
+	{ 0x0b, "COMPRESSION_FAILURE" },
+	{ 0x0c, "UNSUPPORTED_KIND" },
+	{ 0x0d, "REGION_VIOLATION" },
+	{ 0x0e, "BOTH_PTES_VALID" },
+	{ 0x0f, "INFO_TYPE_POISONED" },
+	{}
+};
+
+const struct nvkm_enum
+gk104_fifo_fault_hubclient[] = {
+	{ 0x00, "VIP" },
+	{ 0x01, "CE0" },
+	{ 0x02, "CE1" },
+	{ 0x03, "DNISO" },
+	{ 0x04, "FE" },
+	{ 0x05, "FECS" },
+	{ 0x06, "HOST" },
+	{ 0x07, "HOST_CPU" },
+	{ 0x08, "HOST_CPU_NB" },
+	{ 0x09, "ISO" },
+	{ 0x0a, "MMU" },
+	{ 0x0b, "MSPDEC" },
+	{ 0x0c, "MSPPP" },
+	{ 0x0d, "MSVLD" },
+	{ 0x0e, "NISO" },
+	{ 0x0f, "P2P" },
+	{ 0x10, "PD" },
+	{ 0x11, "PERF" },
+	{ 0x12, "PMU" },
+	{ 0x13, "RASTERTWOD" },
+	{ 0x14, "SCC" },
+	{ 0x15, "SCC_NB" },
+	{ 0x16, "SEC" },
+	{ 0x17, "SSYNC" },
+	{ 0x18, "GR_CE" },
+	{ 0x19, "CE2" },
+	{ 0x1a, "XV" },
+	{ 0x1b, "MMU_NB" },
+	{ 0x1c, "MSENC" },
+	{ 0x1d, "DFALCON" },
+	{ 0x1e, "SKED" },
+	{ 0x1f, "AFALCON" },
+	{}
+};
+
+const struct nvkm_enum
+gk104_fifo_fault_gpcclient[] = {
+	{ 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
+	{ 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
+	{ 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
+	{ 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
+	{ 0x0c, "RAST" },
+	{ 0x0d, "GCC" },
+	{ 0x0e, "GPCCS" },
+	{ 0x0f, "PROP_0" },
+	{ 0x10, "PROP_1" },
+	{ 0x11, "PROP_2" },
+	{ 0x12, "PROP_3" },
+	{ 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
+	{ 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
+	{ 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
+	{ 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
+	{ 0x1f, "GPM" },
+	{ 0x20, "LTP_UTLB_0" },
+	{ 0x21, "LTP_UTLB_1" },
+	{ 0x22, "LTP_UTLB_2" },
+	{ 0x23, "LTP_UTLB_3" },
+	{ 0x24, "GPC_RGG_UTLB" },
+	{}
+};
+
 static void
 gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
 {
@@ -624,40 +810,6 @@ gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo)
 	nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat);
 }
 
-static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = {
-	{ 0x00000001, "MEMREQ" },
-	{ 0x00000002, "MEMACK_TIMEOUT" },
-	{ 0x00000004, "MEMACK_EXTRA" },
-	{ 0x00000008, "MEMDAT_TIMEOUT" },
-	{ 0x00000010, "MEMDAT_EXTRA" },
-	{ 0x00000020, "MEMFLUSH" },
-	{ 0x00000040, "MEMOP" },
-	{ 0x00000080, "LBCONNECT" },
-	{ 0x00000100, "LBREQ" },
-	{ 0x00000200, "LBACK_TIMEOUT" },
-	{ 0x00000400, "LBACK_EXTRA" },
-	{ 0x00000800, "LBDAT_TIMEOUT" },
-	{ 0x00001000, "LBDAT_EXTRA" },
-	{ 0x00002000, "GPFIFO" },
-	{ 0x00004000, "GPPTR" },
-	{ 0x00008000, "GPENTRY" },
-	{ 0x00010000, "GPCRC" },
-	{ 0x00020000, "PBPTR" },
-	{ 0x00040000, "PBENTRY" },
-	{ 0x00080000, "PBCRC" },
-	{ 0x00100000, "XBARCONNECT" },
-	{ 0x00200000, "METHOD" },
-	{ 0x00400000, "METHODCRC" },
-	{ 0x00800000, "DEVICE" },
-	{ 0x02000000, "SEMAPHORE" },
-	{ 0x04000000, "ACQUIRE" },
-	{ 0x08000000, "PRI" },
-	{ 0x20000000, "NO_CTXSW_SEG" },
-	{ 0x40000000, "PBSEG" },
-	{ 0x80000000, "SIGNATURE" },
-	{}
-};
-
 void
 gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit)
 {
@@ -698,36 +850,6 @@ gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit)
 	nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat);
 }
 
-static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = {
-	{ 0x00000001, "HCE_RE_ILLEGAL_OP" },
-	{ 0x00000002, "HCE_RE_ALIGNB" },
-	{ 0x00000004, "HCE_PRIV" },
-	{ 0x00000008, "HCE_ILLEGAL_MTHD" },
-	{ 0x00000010, "HCE_ILLEGAL_CLASS" },
-	{}
-};
-
-void
-gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit)
-{
-	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
-	struct nvkm_device *device = subdev->device;
-	u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000));
-	u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask;
-	u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff;
-	char msg[128];
-
-	if (stat) {
-		nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat);
-		nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n",
-			   unit, stat, msg, chid,
-			   nvkm_rd32(device, 0x040150 + (unit * 0x2000)),
-			   nvkm_rd32(device, 0x040154 + (unit * 0x2000)));
-	}
-
-	nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat);
-}
-
 void
 gk104_fifo_intr_runlist(struct gk104_fifo *fifo)
 {
@@ -899,6 +1021,44 @@ gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data)
 	}
 }
 
+void
+gk104_fifo_init(struct nvkm_fifo *base)
+{
+	struct gk104_fifo *fifo = gk104_fifo(base);
+	struct nvkm_device *device = fifo->base.engine.subdev.device;
+	int i;
+
+	/* Enable PBDMAs. */
+	fifo->func->pbdma->init(fifo);
+
+	/* PBDMA[n] */
+	for (i = 0; i < fifo->pbdma_nr; i++) {
+		nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
+		nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
+		nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
+	}
+
+	/* PBDMA[n].HCE */
+	for (i = 0; i < fifo->pbdma_nr; i++) {
+		nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
+		nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
+	}
+
+	nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
+
+	if (fifo->func->pbdma->init_timeout)
+		fifo->func->pbdma->init_timeout(fifo);
+
+	nvkm_wr32(device, 0x002100, 0xffffffff);
+	nvkm_wr32(device, 0x002140, 0x7fffffff);
+}
+
+int
+gk104_fifo_chid_nr(struct nvkm_fifo *fifo)
+{
+	return 4096;
+}
+
 int
 gk104_fifo_oneinit(struct nvkm_fifo *base)
 {
@@ -990,44 +1150,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 	return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0);
 }
 
-void
-gk104_fifo_init(struct nvkm_fifo *base)
-{
-	struct gk104_fifo *fifo = gk104_fifo(base);
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	int i;
-
-	/* Enable PBDMAs. */
-	fifo->func->pbdma->init(fifo);
-
-	/* PBDMA[n] */
-	for (i = 0; i < fifo->pbdma_nr; i++) {
-		nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
-		nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
-		nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
-	}
-
-	/* PBDMA[n].HCE */
-	for (i = 0; i < fifo->pbdma_nr; i++) {
-		nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */
-		nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */
-	}
-
-	nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12);
-
-	if (fifo->func->pbdma->init_timeout)
-		fifo->func->pbdma->init_timeout(fifo);
-
-	nvkm_wr32(device, 0x002100, 0xffffffff);
-	nvkm_wr32(device, 0x002140, 0x7fffffff);
-}
-
-int
-gk104_fifo_chid_nr(struct nvkm_fifo *fifo)
-{
-	return 4096;
-}
-
 void *
 gk104_fifo_dtor(struct nvkm_fifo *base)
 {
@@ -1077,128 +1199,6 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device,
 	return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, &fifo->base);
 }
 
-const struct nvkm_enum
-gk104_fifo_fault_access[] = {
-	{ 0x0, "READ" },
-	{ 0x1, "WRITE" },
-	{}
-};
-
-const struct nvkm_enum
-gk104_fifo_fault_engine[] = {
-	{ 0x00, "GR", NULL, NVKM_ENGINE_GR },
-	{ 0x01, "DISPLAY" },
-	{ 0x02, "CAPTURE" },
-	{ 0x03, "IFB", NULL, NVKM_ENGINE_IFB },
-	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
-	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
-	{ 0x06, "SCHED" },
-	{ 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO },
-	{ 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO },
-	{ 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO },
-	{ 0x0f, "HOSTSR" },
-	{ 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD },
-	{ 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP },
-	{ 0x13, "PERF" },
-	{ 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC },
-	{ 0x15, "CE0", NULL, NVKM_ENGINE_CE, 0 },
-	{ 0x16, "CE1", NULL, NVKM_ENGINE_CE, 1 },
-	{ 0x17, "PMU" },
-	{ 0x18, "PTP" },
-	{ 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC },
-	{ 0x1b, "CE2", NULL, NVKM_ENGINE_CE, 2 },
-	{}
-};
-
-const struct nvkm_enum
-gk104_fifo_fault_reason[] = {
-	{ 0x00, "PDE" },
-	{ 0x01, "PDE_SIZE" },
-	{ 0x02, "PTE" },
-	{ 0x03, "VA_LIMIT_VIOLATION" },
-	{ 0x04, "UNBOUND_INST_BLOCK" },
-	{ 0x05, "PRIV_VIOLATION" },
-	{ 0x06, "RO_VIOLATION" },
-	{ 0x07, "WO_VIOLATION" },
-	{ 0x08, "PITCH_MASK_VIOLATION" },
-	{ 0x09, "WORK_CREATION" },
-	{ 0x0a, "UNSUPPORTED_APERTURE" },
-	{ 0x0b, "COMPRESSION_FAILURE" },
-	{ 0x0c, "UNSUPPORTED_KIND" },
-	{ 0x0d, "REGION_VIOLATION" },
-	{ 0x0e, "BOTH_PTES_VALID" },
-	{ 0x0f, "INFO_TYPE_POISONED" },
-	{}
-};
-
-const struct nvkm_enum
-gk104_fifo_fault_hubclient[] = {
-	{ 0x00, "VIP" },
-	{ 0x01, "CE0" },
-	{ 0x02, "CE1" },
-	{ 0x03, "DNISO" },
-	{ 0x04, "FE" },
-	{ 0x05, "FECS" },
-	{ 0x06, "HOST" },
-	{ 0x07, "HOST_CPU" },
-	{ 0x08, "HOST_CPU_NB" },
-	{ 0x09, "ISO" },
-	{ 0x0a, "MMU" },
-	{ 0x0b, "MSPDEC" },
-	{ 0x0c, "MSPPP" },
-	{ 0x0d, "MSVLD" },
-	{ 0x0e, "NISO" },
-	{ 0x0f, "P2P" },
-	{ 0x10, "PD" },
-	{ 0x11, "PERF" },
-	{ 0x12, "PMU" },
-	{ 0x13, "RASTERTWOD" },
-	{ 0x14, "SCC" },
-	{ 0x15, "SCC_NB" },
-	{ 0x16, "SEC" },
-	{ 0x17, "SSYNC" },
-	{ 0x18, "GR_CE" },
-	{ 0x19, "CE2" },
-	{ 0x1a, "XV" },
-	{ 0x1b, "MMU_NB" },
-	{ 0x1c, "MSENC" },
-	{ 0x1d, "DFALCON" },
-	{ 0x1e, "SKED" },
-	{ 0x1f, "AFALCON" },
-	{}
-};
-
-const struct nvkm_enum
-gk104_fifo_fault_gpcclient[] = {
-	{ 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
-	{ 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
-	{ 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
-	{ 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },
-	{ 0x0c, "RAST" },
-	{ 0x0d, "GCC" },
-	{ 0x0e, "GPCCS" },
-	{ 0x0f, "PROP_0" },
-	{ 0x10, "PROP_1" },
-	{ 0x11, "PROP_2" },
-	{ 0x12, "PROP_3" },
-	{ 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },
-	{ 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },
-	{ 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },
-	{ 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },
-	{ 0x1f, "GPM" },
-	{ 0x20, "LTP_UTLB_0" },
-	{ 0x21, "LTP_UTLB_1" },
-	{ 0x22, "LTP_UTLB_2" },
-	{ 0x23, "LTP_UTLB_3" },
-	{ 0x24, "GPC_RGG_UTLB" },
-	{}
-};
-
 static const struct gk104_fifo_func
 gk104_fifo = {
 	.chid_nr = gk104_fifo_chid_nr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
index 95a30fbf65a1..f9f8371d3756 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
@@ -49,6 +49,72 @@ nv04_fifo_ramfc[] = {
 	{}
 };
 
+void
+nv04_fifo_dma_fini(struct nvkm_fifo_chan *base)
+{
+	struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
+	struct nv04_fifo *fifo = chan->fifo;
+	struct nvkm_device *device = fifo->base.engine.subdev.device;
+	struct nvkm_memory *fctx = device->imem->ramfc;
+	const struct nv04_fifo_ramfc *c;
+	unsigned long flags;
+	u32 mask = fifo->base.nr - 1;
+	u32 data = chan->ramfc;
+	u32 chid;
+
+	/* prevent fifo context switches */
+	spin_lock_irqsave(&fifo->base.lock, flags);
+	nvkm_wr32(device, NV03_PFIFO_CACHES, 0);
+
+	/* if this channel is active, replace it with a null context */
+	chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & mask;
+	if (chid == chan->base.chid) {
+		nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
+		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0);
+		nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
+
+		c = fifo->ramfc;
+		nvkm_kmap(fctx);
+		do {
+			u32 rm = ((1ULL << c->bits) - 1) << c->regs;
+			u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
+			u32 rv = (nvkm_rd32(device, c->regp) &  rm) >> c->regs;
+			u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm);
+			nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
+		} while ((++c)->bits);
+		nvkm_done(fctx);
+
+		c = fifo->ramfc;
+		do {
+			nvkm_wr32(device, c->regp, 0x00000000);
+		} while ((++c)->bits);
+
+		nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0);
+		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0);
+		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, mask);
+		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1);
+		nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);
+	}
+
+	/* restore normal operation, after disabling dma mode */
+	nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0);
+	nvkm_wr32(device, NV03_PFIFO_CACHES, 1);
+	spin_unlock_irqrestore(&fifo->base.lock, flags);
+}
+
+void
+nv04_fifo_dma_init(struct nvkm_fifo_chan *base)
+{
+	struct nv04_fifo_chan *chan = nv04_fifo_chan(base);
+	struct nv04_fifo *fifo = chan->fifo;
+	struct nvkm_device *device = fifo->base.engine.subdev.device;
+	u32 mask = 1 << chan->base.chid;
+	unsigned long flags;
+	spin_lock_irqsave(&fifo->base.lock, flags);
+	nvkm_mask(device, NV04_PFIFO_MODE, mask, mask);
+	spin_unlock_irqrestore(&fifo->base.lock, flags);
+}
+
 static const struct nvkm_chan_func
 nv04_chan = {
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
index f3f11a324591..101015a1a7e9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c
@@ -64,22 +64,6 @@ nv50_fifo_runlist_update(struct nv50_fifo *fifo)
 	mutex_unlock(&fifo->base.mutex);
 }
 
-int
-nv50_fifo_oneinit(struct nvkm_fifo *base)
-{
-	struct nv50_fifo *fifo = nv50_fifo(base);
-	struct nvkm_device *device = fifo->base.engine.subdev.device;
-	int ret;
-
-	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000,
-			      false, &fifo->runlist[0]);
-	if (ret)
-		return ret;
-
-	return nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000,
-			       false, &fifo->runlist[1]);
-}
-
 void
 nv50_fifo_init(struct nvkm_fifo *base)
 {
@@ -110,6 +94,22 @@ nv50_fifo_chid_nr(struct nvkm_fifo *fifo)
 	return 128;
 }
 
+int
+nv50_fifo_oneinit(struct nvkm_fifo *base)
+{
+	struct nv50_fifo *fifo = nv50_fifo(base);
+	struct nvkm_device *device = fifo->base.engine.subdev.device;
+	int ret;
+
+	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000,
+			      false, &fifo->runlist[0]);
+	if (ret)
+		return ret;
+
+	return nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000,
+			       false, &fifo->runlist[1]);
+}
+
 void *
 nv50_fifo_dtor(struct nvkm_fifo *base)
 {