diff options
| -rw-r--r-- | drivers/gpu/drm/nouveau/include/nvif/class.h | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_chan.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_drm.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nv84_fence.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/device/base.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c | 19 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c | 550 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c | 280 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 8 |
13 files changed, 610 insertions, 276 deletions
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 36c59c7afd95..15529d2c85f9 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -86,6 +86,7 @@ #define PASCAL_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c06f #define VOLTA_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c36f #define TURING_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c46f +#define AMPERE_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c56f #define AMPERE_CHANNEL_GPFIFO_B /* if0020.h */ 0x0000c76f #define NV50_DISP /* if0010.h */ 0x00005070 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index c71f412bbb7f..4825403a2688 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -56,6 +56,7 @@ struct nvkm_fifo { struct { #define NVKM_FIFO_NONSTALL_EVENT BIT(0) struct nvkm_event event; + struct nvkm_inth intr; } nonstall; struct { @@ -92,5 +93,6 @@ int gm200_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gp100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gv100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int tu102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); +int ga100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int ga102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index efd6cf46921b..e648ecd0c1a0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -253,6 +253,7 @@ nouveau_channel_ctor(struct nouveau_drm *drm, struct nvif_device *device, bool p int version; } hosts[] = { { AMPERE_CHANNEL_GPFIFO_B, 0 }, + { AMPERE_CHANNEL_GPFIFO_A, 0 }, { TURING_CHANNEL_GPFIFO_A, 0 }, { VOLTA_CHANNEL_GPFIFO_A, 0 }, { PASCAL_CHANNEL_GPFIFO_A, 0 }, @@ -365,8 +366,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) if (ret) return ret; - if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO && - chan->user.oclass < AMPERE_CHANNEL_GPFIFO_B) { + if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { struct { struct nvif_event_v0 base; struct nvif_chan_event_v0 host; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 61be972f8385..a19f18b251f3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -348,9 +348,6 @@ nouveau_accel_gr_init(struct nouveau_drm *drm) u64 runm; int ret; - if (device->info.family >= NV_DEVICE_INFO_V0_AMPERE) - return; - /* Allocate channel that has access to the graphics engine. */ runm = nvif_fifo_runlist(device, NV_DEVICE_HOST_RUNLIST_ENGINES_GR); if (!runm) { @@ -473,6 +470,7 @@ nouveau_accel_init(struct nouveau_drm *drm) case PASCAL_CHANNEL_GPFIFO_A: case VOLTA_CHANNEL_GPFIFO_A: case TURING_CHANNEL_GPFIFO_A: + case AMPERE_CHANNEL_GPFIFO_A: case AMPERE_CHANNEL_GPFIFO_B: ret = nvc0_fence_create(drm); break; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index a01fd934c85e..812b8c62eeba 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -210,7 +210,7 @@ nv84_fence_create(struct nouveau_drm *drm) priv->base.context_new = nv84_fence_context_new; priv->base.context_del = nv84_fence_context_del; - priv->base.uevent = drm->client.device.info.family < NV_DEVICE_INFO_V0_AMPERE; + priv->base.uevent = true; mutex_init(&priv->mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 490fac3a4a3a..852c18aec4cd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2590,6 +2590,7 @@ nv170_chipset = { .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, .ce = { 0x000003ff, ga100_ce_new }, + .fifo = { 0x00000001, ga100_fifo_new }, }; static const struct nvkm_device_chip diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index f9aad4cf8271..5a074b9970ab 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -23,6 +23,7 @@ nvkm-y += nvkm/engine/fifo/gm200.o nvkm-y += nvkm/engine/fifo/gp100.o nvkm-y += nvkm/engine/fifo/gv100.o nvkm-y += nvkm/engine/fifo/tu102.o +nvkm-y += nvkm/engine/fifo/ga100.o nvkm-y += nvkm/engine/fifo/ga102.o nvkm-y += nvkm/engine/fifo/ucgrp.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 61534a8a33d7..abce1ecfd4d4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -124,6 +124,7 @@ nvkm_fifo_init(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); struct nvkm_runq *runq; + struct nvkm_runl *runl; u32 mask = 0; if (fifo->func->init_pbdmas) { @@ -136,7 +137,13 @@ nvkm_fifo_init(struct nvkm_engine *engine) runq->func->init(runq); } - fifo->func->init(fifo); + nvkm_runl_foreach(runl, fifo) { + if (runl->func->init) + runl->func->init(runl); + } + + if (fifo->func->init) + fifo->func->init(fifo); nvkm_inth_allow(&fifo->engine.subdev.inth); return 0; @@ -243,7 +250,7 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) return ret; nvkm_runl_foreach(runl, fifo) { - RUNL_DEBUG(runl, ""); + RUNL_DEBUG(runl, "chan:%06x", runl->chan); nvkm_runl_foreach_engn(engn, runl) { ENGN_DEBUG(engn, ""); } @@ -259,6 +266,14 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) } } + /* Initialise non-stall intr handling. */ + if (fifo->func->nonstall_ctor) { + ret = fifo->func->nonstall_ctor(fifo); + if (ret) { + nvkm_error(subdev, "nonstall %d\n", ret); + } + } + /* Allocate USERD + BAR1 polling area. */ if (fifo->func->chan.func->userd->bar == 1) { struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c new file mode 100644 index 000000000000..12a5d99d5e77 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c @@ -0,0 +1,550 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" +#include "cgrp.h" +#include "chan.h" +#include "chid.h" +#include "runl.h" +#include "runq.h" + +#include <core/gpuobj.h> +#include <subdev/top.h> +#include <subdev/vfn.h> + +#include <nvif/class.h> + +/*TODO: allocate? */ +#define GA100_FIFO_NONSTALL_VECTOR 0 + +static u32 +ga100_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return (chan->cgrp->runl->doorbell << 16) | chan->id; +} + +static void +ga100_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0x00000003); +} + +static void +ga100_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + const int gfid = 0; + + nvkm_wr32(device, runl->chan + (chan->id * 4), 0x00000002); + nvkm_wr32(device, runl->addr + 0x0090, (gfid << 16) | chan->id); /* INTERNAL_DOORBELL. */ +} + +static void +ga100_chan_unbind(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0xffffffff); +} + +static int +ga100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + const u32 limit2 = ilog2(length / 8); + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x010, 0x0000face); + nvkm_wo32(chan->inst, 0x030, 0x7ffff902); + nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset)); + nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->inst, 0x084, 0x20400000); + nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm); + nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000); + nvkm_wo32(chan->inst, 0x0e8, chan->id); + nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000)); + nvkm_wo32(chan->inst, 0x0f8, 0x80000000 | GA100_FIFO_NONSTALL_VECTOR); + nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); + nvkm_done(chan->inst); + return 0; +} + +static const struct nvkm_chan_func_ramfc +ga100_chan_ramfc = { + .write = ga100_chan_ramfc_write, + .devm = 0xfff, + .priv = true, +}; + +const struct nvkm_chan_func +ga100_chan = { + .inst = &gf100_chan_inst, + .userd = &gv100_chan_userd, + .ramfc = &ga100_chan_ramfc, + .unbind = ga100_chan_unbind, + .start = ga100_chan_start, + .stop = ga100_chan_stop, + .preempt = gk110_chan_preempt, + .doorbell_handle = ga100_chan_doorbell_handle, +}; + +static void +ga100_cgrp_preempt(struct nvkm_cgrp *cgrp) +{ + struct nvkm_runl *runl = cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x01000000 | cgrp->id); +} + +const struct nvkm_cgrp_func +ga100_cgrp = { + .preempt = ga100_cgrp_preempt, +}; + +static int +ga100_engn_cxid(struct nvkm_engn *engn, bool *cgid) +{ + struct nvkm_runl *runl = engn->runl; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u32 stat = nvkm_rd32(device, runl->addr + 0x200 + engn->id * 0x40); + + ENGN_DEBUG(engn, "status %08x", stat); + *cgid = true; + + switch ((stat & 0x0000e000) >> 13) { + case 0 /* INVALID */: return -ENODEV; + case 1 /* VALID */: + case 5 /* SAVE */: return (stat & 0x00000fff); + case 6 /* LOAD */: return (stat & 0x0fff0000) >> 16; + case 7 /* SWITCH */: + if (nvkm_engine_chsw_load(engn->engine)) + return (stat & 0x0fff0000) >> 16; + return (stat & 0x00000fff); + default: + WARN_ON(1); + break; + } + + return -ENODEV; +} + +const struct nvkm_engn_func +ga100_engn = { + .cxid = ga100_engn_cxid, + .ctor = gk104_ectx_ctor, + .bind = gv100_ectx_bind, +}; + +const struct nvkm_engn_func +ga100_engn_ce = { + .cxid = ga100_engn_cxid, + .ctor = gv100_ectx_ce_ctor, + .bind = gv100_ectx_ce_bind, +}; + +static bool +ga100_runq_idle(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + return !(nvkm_rd32(device, 0x04015c + (runq->id * 0x800)) & 0x0000e000); +} + +static bool +ga100_runq_intr_1(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, 0x040180 + (runq->id * 0x800)); + u32 intr = nvkm_rd32(device, 0x040148 + (runq->id * 0x800)); + u32 stat = intr & inte; + + if (!stat) { + RUNQ_DEBUG(runq, "inte1 %08x %08x", intr, inte); + return false; + } + + if (stat & 0x80000000) { + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; + struct nvkm_chan *chan; + unsigned long flags; + + RUNQ_ERROR(runq, "CTXNOTVALID chid:%d", chid); + chan = nvkm_runl_chan_get_chid(runl, chid, &flags); + if (chan) { + nvkm_chan_error(chan, true); + nvkm_chan_put(&chan, flags); + } + + nvkm_mask(device, 0x0400ac + (runq->id * 0x800), 0x00030000, 0x00030000); + stat &= ~0x80000000; + } + + if (stat) { + RUNQ_ERROR(runq, "intr1 %08x", stat); + nvkm_wr32(device, 0x0401a0 + (runq->id * 0x800), stat); + } + + nvkm_wr32(device, 0x040148 + (runq->id * 0x800), intr); + return true; +} + +static bool +ga100_runq_intr_0(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, 0x040170 + (runq->id * 0x800)); + u32 intr = nvkm_rd32(device, 0x040108 + (runq->id * 0x800)); + u32 stat = intr & inte; + + if (!stat) { + RUNQ_DEBUG(runq, "inte0 %08x %08x", intr, inte); + return false; + } + + /*TODO: expand on this when fixing up gf100's version. */ + if (stat & 0xc6afe000) { + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; + struct nvkm_chan *chan; + unsigned long flags; + + RUNQ_ERROR(runq, "intr0 %08x", stat); + chan = nvkm_runl_chan_get_chid(runl, chid, &flags); + if (chan) { + nvkm_chan_error(chan, true); + nvkm_chan_put(&chan, flags); + } + + stat &= ~0xc6afe000; + } + + if (stat) { + RUNQ_ERROR(runq, "intr0 %08x", stat); + nvkm_wr32(device, 0x040190 + (runq->id * 0x800), stat); + } + + nvkm_wr32(device, 0x040108 + (runq->id * 0x800), intr); + return true; +} + +static bool +ga100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + bool intr0 = ga100_runq_intr_0(runq, runl); + bool intr1 = ga100_runq_intr_1(runq, runl); + + return intr0 || intr1; +} + +static void +ga100_runq_init(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x040108 + (runq->id * 0x800), 0xffffffff); /* INTR_0 */ + nvkm_wr32(device, 0x040148 + (runq->id * 0x800), 0xffffffff); /* INTR_1 */ + nvkm_wr32(device, 0x040170 + (runq->id * 0x800), 0xffffffff); /* INTR_0_EN_SET_TREE */ + nvkm_wr32(device, 0x040180 + (runq->id * 0x800), 0xffffffff); /* INTR_1_EN_SET_TREE */ +} + +const struct nvkm_runq_func +ga100_runq = { + .init = ga100_runq_init, + .intr = ga100_runq_intr, + .idle = ga100_runq_idle, +}; + +static bool +ga100_runl_preempt_pending(struct nvkm_runl *runl) +{ + return nvkm_rd32(runl->fifo->engine.subdev.device, runl->addr + 0x098) & 0x00100000; +} + +static void +ga100_runl_preempt(struct nvkm_runl *runl) +{ + nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x00000000); +} + +static void +ga100_runl_allow(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000000); +} + +static void +ga100_runl_block(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000001); +} + +static bool +ga100_runl_pending(struct nvkm_runl *runl) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + + return nvkm_rd32(device, runl->addr + 0x08c) & 0x00008000; +} + +static void +ga100_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; + + nvkm_wr32(device, runl->addr + 0x080, lower_32_bits(addr)); + nvkm_wr32(device, runl->addr + 0x084, upper_32_bits(addr)); + nvkm_wr32(device, runl->addr + 0x088, count); +} + +static irqreturn_t +ga100_runl_intr(struct nvkm_inth *inth) +{ + struct nvkm_runl *runl = container_of(inth, typeof(*runl), inth); + struct nvkm_engn *engn; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, runl->addr + 0x120); + u32 intr = nvkm_rd32(device, runl->addr + 0x100); + u32 stat = intr & inte; + u32 info; + + if (!stat) { + RUNL_DEBUG(runl, "inte %08x %08x", intr, inte); + return IRQ_NONE; + } + + if (stat & 0x00000007) { + nvkm_runl_foreach_engn_cond(engn, runl, stat & BIT(engn->id)) { + info = nvkm_rd32(device, runl->addr + 0x224 + (engn->id * 0x40)); + + tu102_fifo_intr_ctxsw_timeout_info(engn, info); + + nvkm_wr32(device, runl->addr + 0x100, BIT(engn->id)); + stat &= ~BIT(engn->id); + } + } + + if (stat & 0x00000300) { + nvkm_wr32(device, runl->addr + 0x100, stat & 0x00000300); + stat &= ~0x00000300; + } + + if (stat & 0x00010000) { + if (runl->runq[0]) { + if (runl->runq[0]->func->intr(runl->runq[0], runl)) + stat &= ~0x00010000; + } + } + + if (stat & 0x00020000) { + if (runl->runq[1]) { + if (runl->runq[1]->func->intr(runl->runq[1], runl)) + stat &= ~0x00020000; + } + } + + if (stat) { + RUNL_ERROR(runl, "intr %08x", stat); + nvkm_wr32(device, runl->addr + 0x140, stat); + } + + nvkm_wr32(device, runl->addr + 0x180, 0x00000001); + return IRQ_HANDLED; +} + +static void +ga100_runl_fini(struct nvkm_runl *runl) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x300, 0x80000000, 0x00000000); + nvkm_inth_block(&runl->inth); +} + +static void +ga100_runl_init(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_runq *runq; + struct nvkm_device *device = fifo->engine.subdev.device; + int i; + + /* Submit NULL runlist and preempt. */ + nvkm_wr32(device, runl->addr + 0x088, 0x00000000); + runl->func->preempt(runl); + + /* Enable doorbell. */ + nvkm_mask(device, runl->addr + 0x300, 0x80000000, 0x80000000); + + nvkm_wr32(device, runl->addr + 0x100, 0xffffffff); /* INTR_0 */ + nvkm_wr32(device, runl->addr + 0x140, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(0) */ + nvkm_wr32(device, runl->addr + 0x120, 0x000f1307); /* INTR_0_EN_SET_TREE(0) */ + nvkm_wr32(device, runl->addr + 0x148, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(1) */ + nvkm_wr32(device, runl->addr + 0x128, 0x00000000); /* INTR_0_EN_SET_TREE(1) */ + + /* Init PBDMA(s). */ + for (i = 0; i < runl->runq_nr; i++) { + runq = runl->runq[i]; + runq->func->init(runq); + } + + nvkm_inth_allow(&runl->inth); +} + +const struct nvkm_runl_func +ga100_runl = { + .init = ga100_runl_init, + .fini = ga100_runl_fini, + .size = 16, + .update = nv50_runl_update, + .insert_cgrp = gv100_runl_insert_cgrp, + .insert_chan = gv100_runl_insert_chan, + .commit = ga100_runl_commit, + .wait = nv50_runl_wait, + .pending = ga100_runl_pending, + .block = ga100_runl_block, + .allow = ga100_runl_allow, + .preempt = ga100_runl_preempt, + .preempt_pending = ga100_runl_preempt_pending, +}; + +static int +ga100_runl_new(struct nvkm_fifo *fifo, int id, u32 addr, struct nvkm_runl **prunl) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_runl *runl; + u32 chcfg = nvkm_rd32(device, addr + 0x004); + u32 chnum = 1 << (chcfg & 0x0000000f); + u32 chaddr = (chcfg & 0xfffffff0); + u32 dbcfg = nvkm_rd32(device, addr + 0x008); + u32 vector = nvkm_rd32(device, addr + 0x160); + int i, ret; + + runl = *prunl = nvkm_runl_new(fifo, id, addr, chnum); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + for (i = 0; i < 2; i++) { + u32 pbcfg = nvkm_rd32(device, addr + 0x010 + (i * 0x04)); + if (pbcfg & 0x80000000) { + runl->runq[runl->runq_nr] = + nvkm_runq_new(fifo, ((pbcfg & 0x03fffc00) - 0x040000) / 0x800); + if (!runl->runq[runl->runq_nr]) + return -ENOMEM; + + runl->runq_nr++; + } + } + + ret = nvkm_inth_add(&device->vfn->intr, vector & 0x00000fff, NVKM_INTR_PRIO_NORMAL, + &fifo->engine.subdev, ga100_runl_intr, &runl->inth); + if (ret) + return ret; + + runl->chan = chaddr; + runl->doorbell = dbcfg >> 16; + return 0; +} + +static irqreturn_t +ga100_fifo_nonstall_intr(struct nvkm_inth *inth) +{ + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), nonstall.intr); + + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); + return IRQ_HANDLED; +} + +static void +ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + + nvkm_inth_block(&fifo->nonstall.intr); +} + +static void +ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + + nvkm_inth_allow(&fifo->nonstall.intr); +} + +const struct nvkm_event_func +ga100_fifo_nonstall = { + .init = ga100_fifo_nonstall_allow, + .fini = ga100_fifo_nonstall_block, +}; + +int +ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) +{ + return nvkm_inth_add(&fifo->engine.subdev.device->vfn->intr, GA100_FIFO_NONSTALL_VECTOR, + NVKM_INTR_PRIO_NORMAL, &fifo->engine.subdev, ga100_fifo_nonstall_intr, + &fifo->nonstall.intr); +} + +int +ga100_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_top_device *tdev; + struct nvkm_runl *runl; + int id = 0, ret; + + nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) { + runl = nvkm_runl_get(fifo, -1, tdev->runlist); + if (!runl) { + ret = ga100_runl_new(fifo, id++, tdev->runlist, &runl); + if (ret) + return ret; + } + + if (tdev->engine < 0) + continue; + + nvkm_runl_add(runl, tdev->engine, (tdev->type == NVKM_ENGINE_CE) ? + fifo->func->engn_ce : fifo->func->engn, tdev->type, tdev->inst); + } + + return 0; +} + +static const struct nvkm_fifo_func +ga100_fifo = { + .runl_ctor = ga100_fifo_runl_ctor, + .mmu_fault = &tu102_fifo_mmu_fault, + .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall = &ga100_fifo_nonstall, + .runl = &ga100_runl, + .runq = &ga100_runq, + .engn = &ga100_engn, + .engn_ce = &ga100_engn_ce, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, + .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan }, +}; + +int +ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_fifo **pfifo) +{ + return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index b0c7b562bfc8..2cdf5da339b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -19,281 +19,27 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#define ga102_fifo(p) container_of((p), struct ga102_fifo, base.engine) -#define ga102_chan(p) container_of((p), struct ga102_chan, object) -#include <engine/fifo.h> +#include "priv.h" -#include <core/memory.h> -#include <subdev/mmu.h> -#include <subdev/timer.h> -#include <subdev/top.h> - -#include <nvif/cl0080.h> -#include <nvif/if0020.h> #include <nvif/class.h> -struct ga102_fifo { - struct nvkm_fifo base; -}; - -struct ga102_chan { - struct nvkm_object object; - - struct { - u32 runl; - u32 chan; - } ctrl; - - struct nvkm_memory *mthd; - struct nvkm_memory *inst; - struct nvkm_memory *user; - struct nvkm_memory *runl; - - struct nvkm_vmm *vmm; -}; - -static int -ga102_chan_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) -{ - if (index == 0) { - oclass->ctor = nvkm_object_new; - oclass->base = (struct nvkm_sclass) { -1, -1, AMPERE_DMA_COPY_B }; - return 0; - } - - return -EINVAL; -} - -static int -ga102_chan_map(struct nvkm_object *object, void *argv, u32 argc, - enum nvkm_object_map *type, u64 *addr, u64 *size) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - u64 bar2 = nvkm_memory_bar2(chan->user); - - if (bar2 == ~0ULL) - return -EFAULT; - - *type = NVKM_OBJECT_MAP_IO; - *addr = device->func->resource_addr(device, 3) + bar2; - *size = 0x1000; - return 0; -} - -static int -ga102_chan_fini(struct nvkm_object *object, bool suspend) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - - nvkm_wr32(device, chan->ctrl.chan, 0x00000003); - - nvkm_wr32(device, chan->ctrl.runl + 0x098, 0x01000000); - nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, chan->ctrl.runl + 0x098) & 0x00100000)) - break; - ); - - nvkm_wr32(device, chan->ctrl.runl + 0x088, 0); - - nvkm_wr32(device, chan->ctrl.chan, 0xffffffff); - return 0; -} - -static int -ga102_chan_init(struct nvkm_object *object) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - - nvkm_mask(device, chan->ctrl.runl + 0x300, 0x80000000, 0x80000000); - - nvkm_wr32(device, chan->ctrl.runl + 0x080, lower_32_bits(nvkm_memory_addr(chan->runl))); - nvkm_wr32(device, chan->ctrl.runl + 0x084, upper_32_bits(nvkm_memory_addr(chan->runl))); - nvkm_wr32(device, chan->ctrl.runl + 0x088, 2); - - nvkm_wr32(device, chan->ctrl.chan, 0x00000002); - nvkm_wr32(device, chan->ctrl.runl + 0x0090, 0); - return 0; -} - -static void * -ga102_chan_dtor(struct nvkm_object *object) -{ - struct ga102_chan *chan = ga102_chan(object); - - if (chan->vmm) { - nvkm_vmm_part(chan->vmm, chan->inst); - nvkm_vmm_unref(&chan->vmm); - } - - nvkm_memory_unref(&chan->runl); - nvkm_memory_unref(&chan->user); - nvkm_memory_unref(&chan->inst); - nvkm_memory_unref(&chan->mthd); - return chan; -} - -static const struct nvkm_object_func -ga102_chan = { - .dtor = ga102_chan_dtor, - .init = ga102_chan_init, - .fini = ga102_chan_fini, - .map = ga102_chan_map, - .sclass = ga102_chan_sclass, -}; - -static int -ga102_chan_new(struct nvkm_device *device, - const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) -{ - struct nvif_chan_v0 *args = argv; - struct nvkm_top_device *tdev; - struct nvkm_vmm *vmm; - struct nvkm_memory *userd; - struct ga102_chan *chan; - int ret; - - if (argc != sizeof(*args)) - return -ENOSYS; - - vmm = nvkm_uvmm_search(oclass->client, args->vmm); - if (IS_ERR(vmm)) - return PTR_ERR(vmm); - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - - nvkm_object_ctor(&ga102_chan, oclass, &chan->object); - *pobject = &chan->object; - - list_for_each_entry(tdev, &device->top->device, head) { - if (tdev->type == NVKM_ENGINE_CE) { - chan->ctrl.runl = tdev->runlist; - break; - } - } - - if (!chan->ctrl.runl) - return -ENODEV; - - chan->ctrl.chan = nvkm_rd32(device, chan->ctrl.runl + 0x004) & 0xfffffff0; - - args->chid = 0; - args->inst = 0; - args->token = nvkm_rd32(device, chan->ctrl.runl + 0x008) & 0xffff0000; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->mthd); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->inst); - if (ret) - return ret; - - nvkm_kmap(chan->inst); - nvkm_wo32(chan->inst, 0x010, 0x0000face); - nvkm_wo32(chan->inst, 0x030, 0x7ffff902); - nvkm_wo32(chan->inst, 0x048, lower_32_bits(args->offset)); - nvkm_wo32(chan->inst, 0x04c, upper_32_bits(args->offset) | - (order_base_2(args->length / 8) << 16)); - nvkm_wo32(chan->inst, 0x084, 0x20400000); - nvkm_wo32(chan->inst, 0x094, 0x30000001); - nvkm_wo32(chan->inst, 0x0ac, 0x00020000); - nvkm_wo32(chan->inst, 0x0e4, 0x00000000); - nvkm_wo32(chan->inst, 0x0e8, 0); - nvkm_wo32(chan->inst, 0x0f4, 0x00001000); - nvkm_wo32(chan->inst, 0x0f8, 0x10003080); - nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); - nvkm_wo32(chan->inst, 0x220, lower_32_bits(nvkm_memory_bar2(chan->mthd))); - nvkm_wo32(chan->inst, 0x224, upper_32_bits(nvkm_memory_bar2(chan->mthd))); - nvkm_done(chan->inst); - - userd = nvkm_umem_search(oclass->client, args->huserd); - if (IS_ERR(userd)) - return PTR_ERR(userd); - - ret = nvkm_memory_kmap(userd, &chan->user); - nvkm_memory_unref(&userd); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->runl); - if (ret) - return ret; - - nvkm_kmap(chan->runl); - nvkm_wo32(chan->runl, 0x00, 0x80030001); - nvkm_wo32(chan->runl, 0x04, 1); - nvkm_wo32(chan->runl, 0x08, 0); - nvkm_wo32(chan->runl, 0x0c, 0x00000000); - nvkm_wo32(chan->runl, 0x10, lower_32_bits(nvkm_memory_addr(chan->user))); - nvkm_wo32(chan->runl, 0x14, upper_32_bits(nvkm_memory_addr(chan->user))); - nvkm_wo32(chan->runl, 0x18, lower_32_bits(nvkm_memory_addr(chan->inst))); - nvkm_wo32(chan->runl, 0x1c, upper_32_bits(nvkm_memory_addr(chan->inst))); - nvkm_done(chan->runl); - - ret = nvkm_vmm_join(vmm, chan->inst); - if (ret) - return ret; - - chan->vmm = nvkm_vmm_ref(vmm); - return 0; -} - -static const struct nvkm_device_oclass -ga102_chan_oclass = { - .ctor = ga102_chan_new, -}; - -static int -ga102_fifo_sclass(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) -{ - if (index == 0) { - oclass->base = (struct nvkm_sclass) { 0, 0, AMPERE_CHANNEL_GPFIFO_B }; - *class = &ga102_chan_oclass; - return 0; - } - - return 1; -} - -static int -ga102_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) -{ - switch (mthd) { - case NV_DEVICE_HOST_CHANNELS: *data = 1; return 0; - default: - break; - } - - return -ENOSYS; -} - -static void * -ga102_fifo_dtor(struct nvkm_engine *engine) -{ - return ga102_fifo(engine); -} - -static const struct nvkm_engine_func +static const struct nvkm_fifo_func ga102_fifo = { - .dtor = ga102_fifo_dtor, - .info = ga102_fifo_info, - .base.sclass = ga102_fifo_sclass, + .runl_ctor = ga100_fifo_runl_ctor, + .mmu_fault = &tu102_fifo_mmu_fault, + .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall = &ga100_fifo_nonstall, + .runl = &ga100_runl, + .runq = &ga100_runq, + .engn = &ga100_engn, + .engn_ce = &ga100_engn_ce, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, + .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_B }, &ga100_chan }, }; int ga102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - struct ga102_fifo *fifo; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - - nvkm_engine_ctor(&ga102_fifo, device, type, inst, true, &fifo->base.engine); - *pfifo = &fifo->base; - return 0; + return nvkm_fifo_new_(&ga102_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 24125e2d5d4f..4d448be19224 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -37,6 +37,7 @@ struct nvkm_fifo_func { void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); + int (*nonstall_ctor)(struct nvkm_fifo *); const struct nvkm_event_func *nonstall; const struct nvkm_runl_func *runl; @@ -191,6 +192,16 @@ extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; +int ga100_fifo_runl_ctor(struct nvkm_fifo *); +int ga100_fifo_nonstall_ctor(struct nvkm_fifo *); +extern const struct nvkm_event_func ga100_fifo_nonstall; +extern const struct nvkm_runl_func ga100_runl; +extern const struct nvkm_runq_func ga100_runq; +extern const struct nvkm_engn_func ga100_engn; +extern const struct nvkm_engn_func ga100_engn_ce; +extern const struct nvkm_cgrp_func ga100_cgrp; +extern const struct nvkm_chan_func ga100_chan; + int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, void *argv, u32 argc, struct nvkm_object **); int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index e654c1c4f769..fe51c075d6c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -308,6 +308,9 @@ nvkm_runl_block(struct nvkm_runl *runl) void nvkm_runl_fini(struct nvkm_runl *runl) { + if (runl->func->fini) + runl->func->fini(runl); + flush_work(&runl->work); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 5159b64377c5..c93d21bb7bd5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -1,6 +1,6 @@ #ifndef __NVKM_RUNL_H__ #define __NVKM_RUNL_H__ -#include <core/os.h> +#include <core/intr.h> struct nvkm_cctx; struct nvkm_cgrp; struct nvkm_chan; @@ -36,6 +36,8 @@ struct nvkm_engn { struct nvkm_runl { const struct nvkm_runl_func { + void (*init)(struct nvkm_runl *); + void (*fini)(struct nvkm_runl *); int runqs; u8 size; int (*update)(struct nvkm_runl *); @@ -53,6 +55,8 @@ struct nvkm_runl { struct nvkm_fifo *fifo; int id; u32 addr; + u32 chan; + u16 doorbell; struct nvkm_chid *cgid; #define NVKM_CHAN_EVENT_ERRORED BIT(0) @@ -63,6 +67,8 @@ struct nvkm_runl { struct nvkm_runq *runq[2]; int runq_nr; + struct nvkm_inth inth; + struct list_head cgrps; int cgrp_nr; int chan_nr; |