From ba25d977571e1551b7032d6104e49efd6f88f8ad Mon Sep 17 00:00:00 2001 From: Colin Xu Date: Thu, 9 Jul 2020 15:09:56 +0800 Subject: drm/i915/gvt: Do not destroy ppgtt_mm during vGPU D3->D0. When system enters S3 state, device enters D3 state while RAM remains powered. From vGPU/GVT perspective, ppgtt_mm is residual in guest memory during vGPU in D3 state, so that when guest state transits from S3->S0, ppgtt_mm can be re-used and no need rebuild. Previous implementation invalidate and destroy ppgtt_mm at DMLR, regardless the power state transition is S0->S3->S0 (guest suspend or resume) or OFF->S0 (normal boot/reboot), invalidate and destroy ppgtt_mm is unnecessary in the former transition case. The patch saves the vGPU D3/D0 transition state when guest writes the PCI_PM_CTRL in vGPU's configure space, then in later DMLR, GVT can decide whether or not invalidate and destroy ppgtt_mm is required. The d3_entered flags is reset after DMLR. To test this feature, make sure S3 is enabled in QEMU parameters: i440fx: PIIX4_PM.disable_s3=0 q35: ICH9-LPC.disable_s3=0 Also need enable sleep option in guest OS if it's disabled. v2: - Revise commit message to more accurate description. (Kevin) - Split patch by logic. (Zhenyu) Reviewed-by: Zhenyu Wang Signed-off-by: Hang Yuan Signed-off-by: Colin Xu Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200709071002.247960-2-colin.xu@intel.com --- drivers/gpu/drm/i915/gvt/cfg_space.c | 24 ++++++++++++++++++++++++ drivers/gpu/drm/i915/gvt/gtt.c | 2 +- drivers/gpu/drm/i915/gvt/gtt.h | 2 ++ drivers/gpu/drm/i915/gvt/gvt.h | 3 +++ drivers/gpu/drm/i915/gvt/vgpu.c | 17 +++++++++++++++-- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index 072725a448db..ad86c5eb5bba 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -70,6 +70,7 @@ static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off, { u8 *cfg_base = vgpu_cfg_space(vgpu); u8 mask, new, old; + pci_power_t pwr; int i = 0; for (; i < bytes && (off + i < sizeof(pci_cfg_space_rw_bmp)); i++) { @@ -91,6 +92,15 @@ static void vgpu_pci_cfg_mem_write(struct intel_vgpu *vgpu, unsigned int off, /* For other configuration space directly copy as it is. */ if (i < bytes) memcpy(cfg_base + off + i, src + i, bytes - i); + + if (off == vgpu->cfg_space.pmcsr_off && vgpu->cfg_space.pmcsr_off) { + pwr = (pci_power_t __force)(*(u16*)(&vgpu_cfg_space(vgpu)[off]) + & PCI_PM_CTRL_STATE_MASK); + if (pwr == PCI_D3hot) + vgpu->d3_entered = true; + gvt_dbg_core("vgpu-%d power status changed to %d\n", + vgpu->id, pwr); + } } /** @@ -366,6 +376,7 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; u16 *gmch_ctl; + u8 next; memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space, info->cfg_space_size); @@ -401,6 +412,19 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, pci_resource_len(gvt->gt->i915->drm.pdev, 2); memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4); + + /* PM Support */ + vgpu->cfg_space.pmcsr_off = 0; + if (vgpu_cfg_space(vgpu)[PCI_STATUS] & PCI_STATUS_CAP_LIST) { + next = vgpu_cfg_space(vgpu)[PCI_CAPABILITY_LIST]; + do { + if (vgpu_cfg_space(vgpu)[next + PCI_CAP_LIST_ID] == PCI_CAP_ID_PM) { + vgpu->cfg_space.pmcsr_off = next + PCI_PM_CTRL; + break; + } + next = vgpu_cfg_space(vgpu)[next + PCI_CAP_LIST_NEXT]; + } while (next); + } } /** diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 210016192ce7..a3a4305eda01 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -2501,7 +2501,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) return create_scratch_page_tree(vgpu); } -static void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu) +void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu) { struct list_head *pos, *n; struct intel_vgpu_mm *mm; diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index 320b8d6ad92f..52d0d88abd86 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -279,4 +279,6 @@ int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, void *p_data, unsigned int bytes); +void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu); + #endif /* _GVT_GTT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index a4a6db6b7f90..ff7f2515a6fe 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -106,6 +106,7 @@ struct intel_vgpu_pci_bar { struct intel_vgpu_cfg_space { unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE]; struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM]; + u32 pmcsr_off; }; #define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space) @@ -198,6 +199,8 @@ struct intel_vgpu { struct intel_vgpu_submission submission; struct radix_tree_root page_track_tree; u32 hws_pga[I915_NUM_ENGINES]; + /* Set on PCI_D3, reset on DMLR, not reflecting the actual PM state */ + bool d3_entered; struct dentry *debugfs; diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 7d361623ff67..fb12448fe353 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -257,6 +257,7 @@ void intel_gvt_release_vgpu(struct intel_vgpu *vgpu) intel_gvt_deactivate_vgpu(vgpu); mutex_lock(&vgpu->vgpu_lock); + vgpu->d3_entered = false; intel_vgpu_clean_workloads(vgpu, ALL_ENGINES); intel_vgpu_dmabuf_cleanup(vgpu); mutex_unlock(&vgpu->vgpu_lock); @@ -393,6 +394,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); idr_init(&vgpu->object_idr); intel_vgpu_init_cfg_space(vgpu, param->primary); + vgpu->d3_entered = false; ret = intel_vgpu_init_mmio(vgpu); if (ret) @@ -557,10 +559,15 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, /* full GPU reset or device model level reset */ if (engine_mask == ALL_ENGINES || dmlr) { intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); - intel_vgpu_invalidate_ppgtt(vgpu); + if (engine_mask == ALL_ENGINES) + intel_vgpu_invalidate_ppgtt(vgpu); /*fence will not be reset during virtual reset */ if (dmlr) { - intel_vgpu_reset_gtt(vgpu); + if(!vgpu->d3_entered) { + intel_vgpu_invalidate_ppgtt(vgpu); + intel_vgpu_destroy_all_ppgtt_mm(vgpu); + } + intel_vgpu_reset_ggtt(vgpu, true); intel_vgpu_reset_resource(vgpu); } @@ -573,6 +580,12 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, /* only reset the failsafe mode when dmlr reset */ vgpu->failsafe = false; vgpu->pv_notified = false; + /* + * PCI_D0 is set before dmlr, so reset d3_entered here + * after done using. + */ + if(vgpu->d3_entered) + vgpu->d3_entered = false; } } -- cgit v1.2.3 From 9e7c0efadb86ddb58965561bbca638d44792d78f Mon Sep 17 00:00:00 2001 From: Colin Xu Date: Thu, 9 Jul 2020 15:09:57 +0800 Subject: drm/i915/gvt: Do not reset pv_notified when vGPU transit from D3->D0 Unlike full initialization like normal boot, guest driver won't pv_notified GVT when vGPU transit from D3->D0. If pv_notified is reset, later vGPU operations will trigger enter into failsafe mode. Considering the fact that vGPU will at least notify GVT pv_notified once before D3/D0 transition, it's safe to skip reset pv_notified in D3->D0. To test this feature, make sure S3 is enabled in QEMU parameters: i440fx: PIIX4_PM.disable_s3=0 q35: ICH9-LPC.disable_s3=0 Also need enable sleep option in guest OS if it's disabled. v2: - Revise commit message to more accurate description. (Kevin) - Split patch by logic. (Zhenyu) Reviewed-by: Zhenyu Wang Signed-off-by: Hang Yuan Signed-off-by: Colin Xu Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20200709071002.247960-3-colin.xu@intel.com --- drivers/gpu/drm/i915/gvt/vgpu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index fb12448fe353..8fa9b31a2484 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -579,13 +579,14 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, intel_vgpu_reset_cfg_space(vgpu); /* only reset the failsafe mode when dmlr reset */ vgpu->failsafe = false; - vgpu->pv_notified = false; /* * PCI_D0 is set before dmlr, so reset d3_entered here * after done using. */ if(vgpu->d3_entered) vgpu->d3_entered = false; + else + vgpu->pv_notified = false; } } -- cgit v1.2.3