diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
101 files changed, 3740 insertions, 1516 deletions
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 096652921453..a9439b415603 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -1492,11 +1492,10 @@ static void gen11_dsi_get_config(struct intel_encoder *encoder, struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); - intel_dsc_get_config(encoder, pipe_config); - /* FIXME: adapt icl_ddi_clock_get() for DSI and use that? */ pipe_config->port_clock = intel_dpll_get_freq(i915, - pipe_config->shared_dpll); + pipe_config->shared_dpll, + &pipe_config->dpll_hw_state); pipe_config->hw.adjusted_mode.crtc_clock = intel_dsi->pclk; if (intel_dsi->dual_link) diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c index 86be032bcf96..e00fdc47c0eb 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.c +++ b/drivers/gpu/drm/i915/display/intel_atomic.c @@ -133,7 +133,6 @@ int intel_digital_connector_atomic_check(struct drm_connector *conn, struct drm_crtc_state *crtc_state; intel_hdcp_atomic_check(conn, old_state, new_state); - intel_psr_atomic_check(conn, old_state, new_state); if (!new_state->crtc) return 0; @@ -270,14 +269,15 @@ void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state) intel_crtc_put_color_blobs(crtc_state); } -void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state) +void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state, + const struct intel_crtc_state *from_crtc_state) { drm_property_replace_blob(&crtc_state->hw.degamma_lut, - crtc_state->uapi.degamma_lut); + from_crtc_state->uapi.degamma_lut); drm_property_replace_blob(&crtc_state->hw.gamma_lut, - crtc_state->uapi.gamma_lut); + from_crtc_state->uapi.gamma_lut); drm_property_replace_blob(&crtc_state->hw.ctm, - crtc_state->uapi.ctm); + from_crtc_state->uapi.ctm); } /** diff --git a/drivers/gpu/drm/i915/display/intel_atomic.h b/drivers/gpu/drm/i915/display/intel_atomic.h index 285de07011dc..62a3365ed5e6 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.h +++ b/drivers/gpu/drm/i915/display/intel_atomic.h @@ -43,7 +43,8 @@ struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc); void intel_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *state); void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state); -void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state); +void intel_crtc_copy_color_blobs(struct intel_crtc_state *crtc_state, + const struct intel_crtc_state *from_crtc_state); struct drm_atomic_state *intel_atomic_state_alloc(struct drm_device *dev); void intel_atomic_state_free(struct drm_atomic_state *state); void intel_atomic_state_clear(struct drm_atomic_state *state); diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index 3334ff253600..7e9f84b00859 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -247,11 +247,19 @@ static void intel_plane_clear_hw_state(struct intel_plane_state *plane_state) } void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, - const struct intel_plane_state *from_plane_state) + const struct intel_plane_state *from_plane_state, + struct intel_crtc *crtc) { intel_plane_clear_hw_state(plane_state); - plane_state->hw.crtc = from_plane_state->uapi.crtc; + /* + * For the bigjoiner slave uapi.crtc will point at + * the master crtc. So we explicitly assign the right + * slave crtc to hw.crtc. uapi.crtc!=NULL simply indicates + * the plane is logically enabled on the uapi level. + */ + plane_state->hw.crtc = from_plane_state->uapi.crtc ? &crtc->base : NULL; + plane_state->hw.fb = from_plane_state->uapi.fb; if (plane_state->hw.fb) drm_framebuffer_get(plane_state->hw.fb); @@ -263,6 +271,21 @@ void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, plane_state->hw.color_encoding = from_plane_state->uapi.color_encoding; plane_state->hw.color_range = from_plane_state->uapi.color_range; plane_state->hw.scaling_filter = from_plane_state->uapi.scaling_filter; + + plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi); + plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi); +} + +void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state) +{ + intel_plane_clear_hw_state(plane_state); + + memcpy(&plane_state->hw, &from_plane_state->hw, + sizeof(plane_state->hw)); + + if (plane_state->hw.fb) + drm_framebuffer_get(plane_state->hw.fb); } void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, @@ -319,15 +342,16 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_ old_plane_state, new_plane_state); } -static struct intel_crtc * -get_crtc_from_states(const struct intel_plane_state *old_plane_state, - const struct intel_plane_state *new_plane_state) +static struct intel_plane * +intel_crtc_get_plane(struct intel_crtc *crtc, enum plane_id plane_id) { - if (new_plane_state->uapi.crtc) - return to_intel_crtc(new_plane_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_plane *plane; - if (old_plane_state->uapi.crtc) - return to_intel_crtc(old_plane_state->uapi.crtc); + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + if (plane->id == plane_id) + return plane; + } return NULL; } @@ -335,23 +359,37 @@ get_crtc_from_states(const struct intel_plane_state *old_plane_state, int intel_plane_atomic_check(struct intel_atomic_state *state, struct intel_plane *plane) { + struct drm_i915_private *i915 = to_i915(state->base.dev); struct intel_plane_state *new_plane_state = intel_atomic_get_new_plane_state(state, plane); const struct intel_plane_state *old_plane_state = intel_atomic_get_old_plane_state(state, plane); - struct intel_crtc *crtc = - get_crtc_from_states(old_plane_state, new_plane_state); - const struct intel_crtc_state *old_crtc_state; - struct intel_crtc_state *new_crtc_state; + const struct intel_plane_state *new_master_plane_state; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(i915, plane->pipe); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (new_crtc_state && new_crtc_state->bigjoiner_slave) { + struct intel_plane *master_plane = + intel_crtc_get_plane(new_crtc_state->bigjoiner_linked_crtc, + plane->id); + + new_master_plane_state = + intel_atomic_get_new_plane_state(state, master_plane); + } else { + new_master_plane_state = new_plane_state; + } + + intel_plane_copy_uapi_to_hw_state(new_plane_state, + new_master_plane_state, + crtc); - intel_plane_copy_uapi_to_hw_state(new_plane_state, new_plane_state); new_plane_state->uapi.visible = false; - if (!crtc) + if (!new_crtc_state) return 0; - old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc); - new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - return intel_plane_atomic_check_with_state(old_crtc_state, new_crtc_state, old_plane_state, @@ -479,6 +517,63 @@ void i9xx_update_planes_on_crtc(struct intel_atomic_state *state, } } +int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, + struct intel_crtc_state *crtc_state, + int min_scale, int max_scale, + bool can_position) +{ + struct drm_framebuffer *fb = plane_state->hw.fb; + struct drm_rect *src = &plane_state->uapi.src; + struct drm_rect *dst = &plane_state->uapi.dst; + unsigned int rotation = plane_state->hw.rotation; + struct drm_rect clip = {}; + int hscale, vscale; + + if (!fb) { + plane_state->uapi.visible = false; + return 0; + } + + drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation); + + /* Check scaling */ + hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale); + vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale); + if (hscale < 0 || vscale < 0) { + DRM_DEBUG_KMS("Invalid scaling of plane\n"); + drm_rect_debug_print("src: ", src, true); + drm_rect_debug_print("dst: ", dst, false); + return -ERANGE; + } + + if (crtc_state->hw.enable) { + clip.x2 = crtc_state->pipe_src_w; + clip.y2 = crtc_state->pipe_src_h; + } + + /* right side of the image is on the slave crtc, adjust dst to match */ + if (crtc_state->bigjoiner_slave) + drm_rect_translate(dst, -crtc_state->pipe_src_w, 0); + + /* + * FIXME: This might need further adjustment for seamless scaling + * with phase information, for the 2p2 and 2p1 scenarios. + */ + plane_state->uapi.visible = drm_rect_clip_scaled(src, dst, &clip); + + drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation); + + if (!can_position && plane_state->uapi.visible && + !drm_rect_equals(dst, &clip)) { + DRM_DEBUG_KMS("Plane must cover entire CRTC\n"); + drm_rect_debug_print("dst: ", dst, false); + drm_rect_debug_print("clip: ", &clip, false); + return -EINVAL; + } + + return 0; +} + const struct drm_plane_helper_funcs intel_plane_helper_funcs = { .prepare_fb = intel_prepare_plane_fb, .cleanup_fb = intel_cleanup_plane_fb, diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.h b/drivers/gpu/drm/i915/display/intel_atomic_plane.h index 59dd1fbb02ea..5c78a087ed86 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.h +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.h @@ -24,7 +24,10 @@ unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state, unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, - const struct intel_plane_state *from_plane_state); + const struct intel_plane_state *from_plane_state, + struct intel_crtc *crtc); +void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state); void intel_update_plane(struct intel_plane *plane, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); @@ -52,6 +55,10 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, struct intel_plane *plane, bool *need_cdclk_calc); +int intel_atomic_plane_check_clipping(struct intel_plane_state *plane_state, + struct intel_crtc_state *crtc_state, + int min_scale, int max_scale, + bool can_position); void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, struct intel_plane_state *plane_state); diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index 19b16517a502..92940a0c5ef8 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -28,6 +28,7 @@ #include <drm/drm_scdc_helper.h> #include "i915_drv.h" +#include "i915_trace.h" #include "intel_audio.h" #include "intel_combo_phy.h" #include "intel_connector.h" @@ -582,6 +583,34 @@ static const struct cnl_ddi_buf_trans ehl_combo_phy_ddi_translations_dp[] = { { 0x6, 0x7F, 0x3F, 0x00, 0x00 }, /* 900 900 0.0 */ }; +static const struct cnl_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr[] = { + /* NT mV Trans mV db */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 200 200 0.0 */ + { 0x8, 0x7F, 0x38, 0x00, 0x07 }, /* 200 250 1.9 */ + { 0x1, 0x7F, 0x33, 0x00, 0x0C }, /* 200 300 3.5 */ + { 0xA, 0x35, 0x36, 0x00, 0x09 }, /* 200 350 4.9 */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 250 250 0.0 */ + { 0x1, 0x7F, 0x38, 0x00, 0x07 }, /* 250 300 1.6 */ + { 0xA, 0x35, 0x35, 0x00, 0x0A }, /* 250 350 2.9 */ + { 0x1, 0x7F, 0x3F, 0x00, 0x00 }, /* 300 300 0.0 */ + { 0xA, 0x35, 0x38, 0x00, 0x07 }, /* 300 350 1.3 */ + { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ +}; + +static const struct cnl_ddi_buf_trans jsl_combo_phy_ddi_translations_edp_hbr2[] = { + /* NT mV Trans mV db */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 200 200 0.0 */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 200 250 1.9 */ + { 0x1, 0x7F, 0x3D, 0x00, 0x02 }, /* 200 300 3.5 */ + { 0xA, 0x35, 0x38, 0x00, 0x07 }, /* 200 350 4.9 */ + { 0x8, 0x7F, 0x3F, 0x00, 0x00 }, /* 250 250 0.0 */ + { 0x1, 0x7F, 0x3F, 0x00, 0x00 }, /* 250 300 1.6 */ + { 0xA, 0x35, 0x3A, 0x00, 0x05 }, /* 250 350 2.9 */ + { 0x1, 0x7F, 0x3F, 0x00, 0x00 }, /* 300 300 0.0 */ + { 0xA, 0x35, 0x38, 0x00, 0x07 }, /* 300 350 1.3 */ + { 0xA, 0x35, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ +}; + struct icl_mg_phy_ddi_buf_trans { u32 cri_txdeemph_override_11_6; u32 cri_txdeemph_override_5_0; @@ -1163,6 +1192,57 @@ ehl_get_combo_buf_trans(struct intel_encoder *encoder, } static const struct cnl_ddi_buf_trans * +jsl_get_combo_buf_trans_hdmi(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_hdmi); + return icl_combo_phy_ddi_translations_hdmi; +} + +static const struct cnl_ddi_buf_trans * +jsl_get_combo_buf_trans_dp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + *n_entries = ARRAY_SIZE(icl_combo_phy_ddi_translations_dp_hbr2); + return icl_combo_phy_ddi_translations_dp_hbr2; +} + +static const struct cnl_ddi_buf_trans * +jsl_get_combo_buf_trans_edp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + + if (dev_priv->vbt.edp.low_vswing) { + if (crtc_state->port_clock > 270000) { + *n_entries = ARRAY_SIZE(jsl_combo_phy_ddi_translations_edp_hbr2); + return jsl_combo_phy_ddi_translations_edp_hbr2; + } else { + *n_entries = ARRAY_SIZE(jsl_combo_phy_ddi_translations_edp_hbr); + return jsl_combo_phy_ddi_translations_edp_hbr; + } + } + + return jsl_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); +} + +static const struct cnl_ddi_buf_trans * +jsl_get_combo_buf_trans(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + int *n_entries) +{ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + return jsl_get_combo_buf_trans_hdmi(encoder, crtc_state, n_entries); + else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP)) + return jsl_get_combo_buf_trans_edp(encoder, crtc_state, n_entries); + else + return jsl_get_combo_buf_trans_dp(encoder, crtc_state, n_entries); +} + +static const struct cnl_ddi_buf_trans * tgl_get_combo_buf_trans_hdmi(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, int *n_entries) @@ -1676,7 +1756,8 @@ static void intel_ddi_clock_get(struct intel_encoder *encoder, encoder->port); else pipe_config->port_clock = - intel_dpll_get_freq(dev_priv, pipe_config->shared_dpll); + intel_dpll_get_freq(dev_priv, pipe_config->shared_dpll, + &pipe_config->dpll_hw_state); ddi_dotclock_get(pipe_config); } @@ -2216,13 +2297,6 @@ static void intel_ddi_get_power_domains(struct intel_encoder *encoder, intel_phy_is_tc(dev_priv, phy)) intel_display_power_get(dev_priv, intel_ddi_main_link_aux_domain(dig_port)); - - /* - * VDSC power is needed when DSC is enabled - */ - if (crtc_state->dsc.compression_enable) - intel_display_power_get(dev_priv, - intel_dsc_power_domain(crtc_state)); } void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder, @@ -2363,7 +2437,9 @@ static u8 intel_ddi_dp_voltage_max(struct intel_dp *intel_dp, else tgl_get_dkl_buf_trans(encoder, crtc_state, &n_entries); } else if (INTEL_GEN(dev_priv) == 11) { - if (IS_JSL_EHL(dev_priv)) + if (IS_PLATFORM(dev_priv, INTEL_JASPERLAKE)) + jsl_get_combo_buf_trans(encoder, crtc_state, &n_entries); + else if (IS_PLATFORM(dev_priv, INTEL_ELKHARTLAKE)) ehl_get_combo_buf_trans(encoder, crtc_state, &n_entries); else if (intel_phy_is_combo(dev_priv, phy)) icl_get_combo_buf_trans(encoder, crtc_state, &n_entries); @@ -2544,7 +2620,9 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder, if (INTEL_GEN(dev_priv) >= 12) ddi_translations = tgl_get_combo_buf_trans(encoder, crtc_state, &n_entries); - else if (IS_JSL_EHL(dev_priv)) + else if (IS_PLATFORM(dev_priv, INTEL_JASPERLAKE)) + ddi_translations = jsl_get_combo_buf_trans(encoder, crtc_state, &n_entries); + else if (IS_PLATFORM(dev_priv, INTEL_ELKHARTLAKE)) ddi_translations = ehl_get_combo_buf_trans(encoder, crtc_state, &n_entries); else ddi_translations = icl_get_combo_buf_trans(encoder, crtc_state, &n_entries); @@ -2970,6 +3048,40 @@ static u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, return 0; } +static void dg1_map_plls_to_ports(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_shared_dpll *pll = crtc_state->shared_dpll; + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + u32 val; + + /* + * If we fail this, something went very wrong: first 2 PLLs should be + * used by first 2 phys and last 2 PLLs by last phys + */ + if (drm_WARN_ON(&dev_priv->drm, + (pll->info->id < DPLL_ID_DG1_DPLL2 && phy >= PHY_C) || + (pll->info->id >= DPLL_ID_DG1_DPLL2 && phy < PHY_C))) + return; + + mutex_lock(&dev_priv->dpll.lock); + + val = intel_de_read(dev_priv, DG1_DPCLKA_CFGCR0(phy)); + drm_WARN_ON(&dev_priv->drm, + (val & DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)) == 0); + + val &= ~DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + val |= DG1_DPCLKA_CFGCR0_DDI_CLK_SEL(pll->info->id, phy); + intel_de_write(dev_priv, DG1_DPCLKA_CFGCR0(phy), val); + intel_de_posting_read(dev_priv, DG1_DPCLKA_CFGCR0(phy)); + + val &= ~DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); + intel_de_write(dev_priv, DG1_DPCLKA_CFGCR0(phy), val); + + mutex_unlock(&dev_priv->dpll.lock); +} + static void icl_map_plls_to_ports(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { @@ -3017,6 +3129,19 @@ static void icl_map_plls_to_ports(struct intel_encoder *encoder, mutex_unlock(&dev_priv->dpll.lock); } +static void dg1_unmap_plls_to_ports(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + + mutex_lock(&dev_priv->dpll.lock); + + intel_de_rmw(dev_priv, DG1_DPCLKA_CFGCR0(phy), 0, + DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy)); + + mutex_unlock(&dev_priv->dpll.lock); +} + static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); @@ -3032,6 +3157,37 @@ static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpll.lock); } +static void dg1_sanitize_port_clk_off(struct drm_i915_private *dev_priv, + u32 port_mask, bool ddi_clk_needed) +{ + enum port port; + u32 val; + + for_each_port_masked(port, port_mask) { + enum phy phy = intel_port_to_phy(dev_priv, port); + bool ddi_clk_off; + + val = intel_de_read(dev_priv, DG1_DPCLKA_CFGCR0(phy)); + ddi_clk_off = val & DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); + + if (ddi_clk_needed == !ddi_clk_off) + continue; + + /* + * Punt on the case now where clock is gated, but it would + * be needed by the port. Something else is really broken then. + */ + if (drm_WARN_ON(&dev_priv->drm, ddi_clk_needed)) + continue; + + drm_notice(&dev_priv->drm, + "PHY %c is disabled with an ungated DDI clock, gate it\n", + phy_name(phy)); + val |= DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); + intel_de_write(dev_priv, DG1_DPCLKA_CFGCR0(phy), val); + } +} + static void icl_sanitize_port_clk_off(struct drm_i915_private *dev_priv, u32 port_mask, bool ddi_clk_needed) { @@ -3114,7 +3270,10 @@ void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) ddi_clk_needed = false; } - icl_sanitize_port_clk_off(dev_priv, port_mask, ddi_clk_needed); + if (IS_DG1(dev_priv)) + dg1_sanitize_port_clk_off(dev_priv, port_mask, ddi_clk_needed); + else + icl_sanitize_port_clk_off(dev_priv, port_mask, ddi_clk_needed); } static void intel_ddi_clk_select(struct intel_encoder *encoder, @@ -3507,7 +3666,8 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, /* 7.l Configure and enable FEC if needed */ intel_ddi_enable_fec(encoder, crtc_state); - intel_dsc_enable(encoder, crtc_state); + if (!crtc_state->bigjoiner) + intel_dsc_enable(encoder, crtc_state); } static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, @@ -3579,7 +3739,8 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, if (!is_mst) intel_ddi_enable_pipe_clock(encoder, crtc_state); - intel_dsc_enable(encoder, crtc_state); + if (!crtc_state->bigjoiner) + intel_dsc_enable(encoder, crtc_state); } static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, @@ -3666,7 +3827,9 @@ static void intel_ddi_pre_enable(struct intel_atomic_state *state, drm_WARN_ON(&dev_priv->drm, crtc_state->has_pch_encoder); - if (INTEL_GEN(dev_priv) >= 11) + if (IS_DG1(dev_priv)) + dg1_map_plls_to_ports(encoder, crtc_state); + else if (INTEL_GEN(dev_priv) >= 11) icl_map_plls_to_ports(encoder, crtc_state); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); @@ -3828,6 +3991,21 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state, ilk_pfit_disable(old_crtc_state); } + if (old_crtc_state->bigjoiner_linked_crtc) { + struct intel_atomic_state *state = + to_intel_atomic_state(old_crtc_state->uapi.state); + struct intel_crtc *slave = + old_crtc_state->bigjoiner_linked_crtc; + const struct intel_crtc_state *old_slave_crtc_state = + intel_atomic_get_old_crtc_state(state, slave); + + intel_crtc_vblank_off(old_slave_crtc_state); + trace_intel_pipe_disable(slave); + + intel_dsc_disable(old_slave_crtc_state); + skl_scaler_disable(old_slave_crtc_state); + } + /* * When called from DP MST code: * - old_conn_state will be NULL @@ -3848,7 +4026,9 @@ static void intel_ddi_post_disable(struct intel_atomic_state *state, intel_ddi_post_disable_dp(state, encoder, old_crtc_state, old_conn_state); - if (INTEL_GEN(dev_priv) >= 11) + if (IS_DG1(dev_priv)) + dg1_unmap_plls_to_ports(encoder); + else if (INTEL_GEN(dev_priv) >= 11) icl_unmap_plls_to_ports(encoder); if (intel_crtc_has_dp_encoder(old_crtc_state) || is_tc_port) @@ -4044,7 +4224,8 @@ static void intel_enable_ddi(struct intel_atomic_state *state, { drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder); - intel_ddi_enable_transcoder_func(encoder, crtc_state); + if (!crtc_state->bigjoiner_slave) + intel_ddi_enable_transcoder_func(encoder, crtc_state); intel_enable_pipe(crtc_state); @@ -4396,20 +4577,14 @@ static void bdw_get_trans_port_sync_config(struct intel_crtc_state *crtc_state) crtc_state->sync_mode_slaves_mask); } -void intel_ddi_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) +static void intel_ddi_read_func_ctl(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->uapi.crtc); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; u32 temp, flags = 0; - /* XXX: DSI transcoder paranoia */ - if (drm_WARN_ON(&dev_priv->drm, transcoder_is_dsi(cpu_transcoder))) - return; - - intel_dsc_get_config(encoder, pipe_config); - temp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); if (temp & TRANS_DDI_PHSYNC) flags |= DRM_MODE_FLAG_PHSYNC; @@ -4503,6 +4678,30 @@ void intel_ddi_get_config(struct intel_encoder *encoder, default: break; } +} + +void intel_ddi_get_config(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; + + /* XXX: DSI transcoder paranoia */ + if (drm_WARN_ON(&dev_priv->drm, transcoder_is_dsi(cpu_transcoder))) + return; + + if (pipe_config->bigjoiner_slave) { + /* read out pipe settings from master */ + enum transcoder save = pipe_config->cpu_transcoder; + + /* Our own transcoder needs to be disabled when reading it in intel_ddi_read_func_ctl() */ + WARN_ON(pipe_config->output_types); + pipe_config->cpu_transcoder = (enum transcoder)pipe_config->bigjoiner_linked_crtc->pipe; + intel_ddi_read_func_ctl(encoder, pipe_config); + pipe_config->cpu_transcoder = save; + } else { + intel_ddi_read_func_ctl(encoder, pipe_config); + } pipe_config->has_audio = intel_ddi_is_audio_enabled(dev_priv, cpu_transcoder); @@ -4528,7 +4727,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder, dev_priv->vbt.edp.bpp = pipe_config->pipe_bpp; } - intel_ddi_clock_get(encoder, pipe_config); + if (!pipe_config->bigjoiner_slave) + intel_ddi_clock_get(encoder, pipe_config); if (IS_GEN9_LP(dev_priv)) pipe_config->lane_lat_optim_mask = @@ -5126,6 +5326,9 @@ static enum hpd_pin cnl_hpd_pin(struct drm_i915_private *dev_priv, return HPD_PORT_A + port - PORT_A; } +#define port_tc_name(port) ((port) - PORT_TC1 + '1') +#define tc_port_name(tc_port) ((tc_port) - TC_PORT_1 + '1') + void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) { struct intel_digital_port *dig_port; @@ -5181,9 +5384,9 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) DRM_MODE_ENCODER_TMDS, "DDI %s%c/PHY %s%c", port >= PORT_TC1 ? "TC" : "", - port >= PORT_TC1 ? port_name(port) : port - PORT_TC1 + '1', + port >= PORT_TC1 ? port_tc_name(port) : port_name(port), tc_port != TC_PORT_NONE ? "TC" : "", - tc_port != TC_PORT_NONE ? phy_name(phy) : tc_port - TC_PORT_1 + '1'); + tc_port != TC_PORT_NONE ? tc_port_name(tc_port) : phy_name(phy)); } else if (INTEL_GEN(dev_priv) >= 11) { enum tc_port tc_port = intel_port_to_tc(dev_priv, port); @@ -5193,7 +5396,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) port_name(port), port >= PORT_C ? " (TC)" : "", tc_port != TC_PORT_NONE ? "TC" : "", - tc_port != TC_PORT_NONE ? phy_name(phy) : tc_port - TC_PORT_1 + '1'); + tc_port != TC_PORT_NONE ? tc_port_name(tc_port) : phy_name(phy)); } else { drm_encoder_init(&dev_priv->drm, &encoder->base, &intel_ddi_funcs, DRM_MODE_ENCODER_TMDS, diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index cddbda5303ff..53a00cf3fa32 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -3631,6 +3631,8 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, struct intel_plane *intel_plane = to_intel_plane(primary); struct intel_plane_state *intel_state = to_intel_plane_state(plane_state); + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(intel_crtc->base.state); struct drm_framebuffer *fb; struct i915_vma *vma; @@ -3653,7 +3655,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, if (c == &intel_crtc->base) continue; - if (!to_intel_crtc(c)->active) + if (!to_intel_crtc_state(c->state)->uapi.active) continue; state = to_intel_plane_state(c->primary->state); @@ -3675,6 +3677,11 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, * pretend the BIOS never had it enabled. */ intel_plane_disable_noatomic(intel_crtc, intel_plane); + if (crtc_state->bigjoiner) { + struct intel_crtc *slave = + crtc_state->bigjoiner_linked_crtc; + intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary)); + } return; @@ -3711,7 +3718,8 @@ valid_fb: drm_framebuffer_get(fb); plane_state->crtc = &intel_crtc->base; - intel_plane_copy_uapi_to_hw_state(intel_state, intel_state); + intel_plane_copy_uapi_to_hw_state(intel_state, intel_state, + intel_crtc); intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB); @@ -3719,127 +3727,6 @@ valid_fb: &to_intel_frontbuffer(fb)->bits); } -static int skl_max_plane_width(const struct drm_framebuffer *fb, - int color_plane, - unsigned int rotation) -{ - int cpp = fb->format->cpp[color_plane]; - - switch (fb->modifier) { - case DRM_FORMAT_MOD_LINEAR: - case I915_FORMAT_MOD_X_TILED: - /* - * Validated limit is 4k, but has 5k should - * work apart from the following features: - * - Ytile (already limited to 4k) - * - FP16 (already limited to 4k) - * - render compression (already limited to 4k) - * - KVMR sprite and cursor (don't care) - * - horizontal panning (TODO verify this) - * - pipe and plane scaling (TODO verify this) - */ - if (cpp == 8) - return 4096; - else - return 5120; - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Yf_TILED_CCS: - case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: - /* FIXME AUX plane? */ - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Yf_TILED: - if (cpp == 8) - return 2048; - else - return 4096; - default: - MISSING_CASE(fb->modifier); - return 2048; - } -} - -static int glk_max_plane_width(const struct drm_framebuffer *fb, - int color_plane, - unsigned int rotation) -{ - int cpp = fb->format->cpp[color_plane]; - - switch (fb->modifier) { - case DRM_FORMAT_MOD_LINEAR: - case I915_FORMAT_MOD_X_TILED: - if (cpp == 8) - return 4096; - else - return 5120; - case I915_FORMAT_MOD_Y_TILED_CCS: - case I915_FORMAT_MOD_Yf_TILED_CCS: - /* FIXME AUX plane? */ - case I915_FORMAT_MOD_Y_TILED: - case I915_FORMAT_MOD_Yf_TILED: - if (cpp == 8) - return 2048; - else - return 5120; - default: - MISSING_CASE(fb->modifier); - return 2048; - } -} - -static int icl_min_plane_width(const struct drm_framebuffer *fb) -{ - /* Wa_14011264657, Wa_14011050563: gen11+ */ - switch (fb->format->format) { - case DRM_FORMAT_C8: - return 18; - case DRM_FORMAT_RGB565: - return 10; - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ARGB8888: - case DRM_FORMAT_ABGR8888: - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_XBGR2101010: - case DRM_FORMAT_ARGB2101010: - case DRM_FORMAT_ABGR2101010: - case DRM_FORMAT_XVYU2101010: - case DRM_FORMAT_Y212: - case DRM_FORMAT_Y216: - return 6; - case DRM_FORMAT_NV12: - return 20; - case DRM_FORMAT_P010: - case DRM_FORMAT_P012: - case DRM_FORMAT_P016: - return 12; - case DRM_FORMAT_XRGB16161616F: - case DRM_FORMAT_XBGR16161616F: - case DRM_FORMAT_ARGB16161616F: - case DRM_FORMAT_ABGR16161616F: - case DRM_FORMAT_XVYU12_16161616: - case DRM_FORMAT_XVYU16161616: - return 4; - default: - return 1; - } -} - -static int icl_max_plane_width(const struct drm_framebuffer *fb, - int color_plane, - unsigned int rotation) -{ - return 5120; -} - -static int skl_max_plane_height(void) -{ - return 4096; -} - -static int icl_max_plane_height(void) -{ - return 4320; -} static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state, @@ -3897,35 +3784,55 @@ intel_plane_fence_y_offset(const struct intel_plane_state *plane_state) return y; } +static int intel_plane_min_width(struct intel_plane *plane, + const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + if (plane->min_width) + return plane->min_width(fb, color_plane, rotation); + else + return 1; +} + +static int intel_plane_max_width(struct intel_plane *plane, + const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + if (plane->max_width) + return plane->max_width(fb, color_plane, rotation); + else + return INT_MAX; +} + +static int intel_plane_max_height(struct intel_plane *plane, + const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + if (plane->max_height) + return plane->max_height(fb, color_plane, rotation); + else + return INT_MAX; +} + static int skl_check_main_surface(struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev); + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); const struct drm_framebuffer *fb = plane_state->hw.fb; unsigned int rotation = plane_state->hw.rotation; int x = plane_state->uapi.src.x1 >> 16; int y = plane_state->uapi.src.y1 >> 16; int w = drm_rect_width(&plane_state->uapi.src) >> 16; int h = drm_rect_height(&plane_state->uapi.src) >> 16; - int max_width, min_width, max_height; - u32 alignment, offset; + int min_width = intel_plane_min_width(plane, fb, 0, rotation); + int max_width = intel_plane_max_width(plane, fb, 0, rotation); + int max_height = intel_plane_max_height(plane, fb, 0, rotation); int aux_plane = intel_main_to_aux_plane(fb, 0); u32 aux_offset = plane_state->color_plane[aux_plane].offset; - - if (INTEL_GEN(dev_priv) >= 11) { - max_width = icl_max_plane_width(fb, 0, rotation); - min_width = icl_min_plane_width(fb); - } else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { - max_width = glk_max_plane_width(fb, 0, rotation); - min_width = 1; - } else { - max_width = skl_max_plane_width(fb, 0, rotation); - min_width = 1; - } - - if (INTEL_GEN(dev_priv) >= 11) - max_height = icl_max_plane_height(); - else - max_height = skl_max_plane_height(); + u32 alignment, offset; if (w > max_width || w < min_width || h > max_height) { drm_dbg_kms(&dev_priv->drm, @@ -4008,22 +3915,19 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state) { - struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *i915 = to_i915(plane->base.dev); const struct drm_framebuffer *fb = plane_state->hw.fb; unsigned int rotation = plane_state->hw.rotation; int uv_plane = 1; - int max_width = skl_max_plane_width(fb, uv_plane, rotation); - int max_height = 4096; + int max_width = intel_plane_max_width(plane, fb, uv_plane, rotation); + int max_height = intel_plane_max_height(plane, fb, uv_plane, rotation); int x = plane_state->uapi.src.x1 >> 17; int y = plane_state->uapi.src.y1 >> 17; int w = drm_rect_width(&plane_state->uapi.src) >> 17; int h = drm_rect_height(&plane_state->uapi.src) >> 17; u32 offset; - intel_add_fb_offsets(&x, &y, plane_state, uv_plane); - offset = intel_plane_compute_aligned_offset(&x, &y, - plane_state, uv_plane); - /* FIXME not quite sure how/if these apply to the chroma plane */ if (w > max_width || h > max_height) { drm_dbg_kms(&i915->drm, @@ -4032,6 +3936,10 @@ static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state) return -EINVAL; } + intel_add_fb_offsets(&x, &y, plane_state, uv_plane); + offset = intel_plane_compute_aligned_offset(&x, &y, + plane_state, uv_plane); + if (is_ccs_modifier(fb->modifier)) { int ccs_plane = main_to_ccs_plane(fb, uv_plane); u32 aux_offset = plane_state->color_plane[ccs_plane].offset; @@ -4407,12 +4315,10 @@ i9xx_plane_check(struct intel_crtc_state *crtc_state, if (ret) return ret; - ret = drm_atomic_helper_check_plane_state(&plane_state->uapi, - &crtc_state->uapi, - DRM_PLANE_HELPER_NO_SCALING, - DRM_PLANE_HELPER_NO_SCALING, - i9xx_plane_has_windowing(plane), - true); + ret = intel_atomic_plane_check_clipping(plane_state, crtc_state, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + i9xx_plane_has_windowing(plane)); if (ret) return ret; @@ -4951,13 +4857,16 @@ static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv) intel_has_gpu_reset(&dev_priv->gt)); } -void intel_prepare_reset(struct drm_i915_private *dev_priv) +void intel_display_prepare_reset(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; struct drm_atomic_state *state; int ret; + if (!HAS_DISPLAY(dev_priv)) + return; + /* reset doesn't touch the display */ if (!dev_priv->params.force_reset_modeset_test && !gpu_reset_clobbers_display(dev_priv)) @@ -5011,13 +4920,16 @@ void intel_prepare_reset(struct drm_i915_private *dev_priv) state->acquire_ctx = ctx; } -void intel_finish_reset(struct drm_i915_private *dev_priv) +void intel_display_finish_reset(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; struct drm_atomic_state *state; int ret; + if (!HAS_DISPLAY(dev_priv)) + return; + /* reset doesn't touch the display */ if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags)) return; @@ -6167,18 +6079,16 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, static int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state) { - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; int width, height; if (crtc_state->pch_pfit.enabled) { width = drm_rect_width(&crtc_state->pch_pfit.dst); height = drm_rect_height(&crtc_state->pch_pfit.dst); } else { - width = adjusted_mode->crtc_hdisplay; - height = adjusted_mode->crtc_vdisplay; + width = pipe_mode->crtc_hdisplay; + height = pipe_mode->crtc_vdisplay; } - return skl_update_scaler(crtc_state, !crtc_state->hw.active, SKL_CRTC_INDEX, &crtc_state->scaler_state.scaler_id, @@ -7234,6 +7144,45 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state) intel_de_write(dev_priv, reg, val); } +static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state, + const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *master = to_intel_crtc(crtc_state->uapi.crtc); + struct intel_crtc_state *master_crtc_state; + struct drm_connector_state *conn_state; + struct drm_connector *conn; + struct intel_encoder *encoder = NULL; + int i; + + if (crtc_state->bigjoiner_slave) + master = crtc_state->bigjoiner_linked_crtc; + + master_crtc_state = intel_atomic_get_new_crtc_state(state, master); + + for_each_new_connector_in_state(&state->base, conn, conn_state, i) { + if (conn_state->crtc != &master->base) + continue; + + encoder = to_intel_encoder(conn_state->best_encoder); + break; + } + + if (!crtc_state->bigjoiner_slave) { + /* need to enable VDSC, which we skipped in pre-enable */ + intel_dsc_enable(encoder, crtc_state); + } else { + /* + * Enable sequence steps 1-7 on bigjoiner master + */ + intel_encoders_pre_pll_enable(state, master); + intel_enable_shared_dpll(master_crtc_state); + intel_encoders_pre_enable(state, master); + + /* and DSC on slave */ + intel_dsc_enable(NULL, crtc_state); + } +} + static void hsw_crtc_enable(struct intel_atomic_state *state, struct intel_crtc *crtc) { @@ -7247,34 +7196,37 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; - intel_encoders_pre_pll_enable(state, crtc); - - if (new_crtc_state->shared_dpll) - intel_enable_shared_dpll(new_crtc_state); + if (!new_crtc_state->bigjoiner) { + intel_encoders_pre_pll_enable(state, crtc); - intel_encoders_pre_enable(state, crtc); + if (new_crtc_state->shared_dpll) + intel_enable_shared_dpll(new_crtc_state); - if (!transcoder_is_dsi(cpu_transcoder)) - intel_set_transcoder_timings(new_crtc_state); + intel_encoders_pre_enable(state, crtc); + } else { + icl_ddi_bigjoiner_pre_enable(state, new_crtc_state); + } intel_set_pipe_src_size(new_crtc_state); + if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) + bdw_set_pipemisc(new_crtc_state); - if (cpu_transcoder != TRANSCODER_EDP && - !transcoder_is_dsi(cpu_transcoder)) - intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder), - new_crtc_state->pixel_multiplier - 1); + if (!new_crtc_state->bigjoiner_slave && !transcoder_is_dsi(cpu_transcoder)) { + intel_set_transcoder_timings(new_crtc_state); - if (new_crtc_state->has_pch_encoder) - intel_cpu_transcoder_set_m_n(new_crtc_state, - &new_crtc_state->fdi_m_n, NULL); + if (cpu_transcoder != TRANSCODER_EDP) + intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder), + new_crtc_state->pixel_multiplier - 1); + + if (new_crtc_state->has_pch_encoder) + intel_cpu_transcoder_set_m_n(new_crtc_state, + &new_crtc_state->fdi_m_n, NULL); - if (!transcoder_is_dsi(cpu_transcoder)) { hsw_set_frame_start_delay(new_crtc_state); - hsw_set_pipeconf(new_crtc_state); } - if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) - bdw_set_pipemisc(new_crtc_state); + if (!transcoder_is_dsi(cpu_transcoder)) + hsw_set_pipeconf(new_crtc_state); crtc->active = true; @@ -7310,6 +7262,11 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, if (INTEL_GEN(dev_priv) >= 11) icl_pipe_mbus_enable(crtc); + if (new_crtc_state->bigjoiner_slave) { + trace_intel_pipe_enable(crtc); + intel_crtc_vblank_on(new_crtc_state); + } + intel_encoders_enable(state, crtc); if (psl_clkgate_wa) { @@ -7439,7 +7396,7 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) { if (phy == PHY_NONE) return false; - else if (IS_ROCKETLAKE(dev_priv)) + else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) return phy <= PHY_D; else if (IS_JSL_EHL(dev_priv)) return phy <= PHY_C; @@ -7451,7 +7408,7 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy) { - if (IS_ROCKETLAKE(dev_priv)) + if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) return false; else if (INTEL_GEN(dev_priv) >= 12) return phy >= PHY_D && phy <= PHY_I; @@ -7463,7 +7420,7 @@ bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy) enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port) { - if (IS_ROCKETLAKE(i915) && port >= PORT_TC1) + if ((IS_DG1(i915) || IS_ROCKETLAKE(i915)) && port >= PORT_TC1) return PHY_C + port - PORT_TC1; else if (IS_JSL_EHL(i915) && port == PORT_D) return PHY_A; @@ -7604,6 +7561,9 @@ static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state) if (crtc_state->shared_dpll) mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE); + if (crtc_state->dsc.compression_enable) + mask |= BIT_ULL(intel_dsc_power_domain(crtc_state)); + return mask; } @@ -8192,7 +8152,7 @@ static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state) { - u32 pixel_rate = crtc_state->hw.adjusted_mode.crtc_clock; + u32 pixel_rate = crtc_state->hw.pipe_mode.crtc_clock; unsigned int pipe_w, pipe_h, pfit_w, pfit_h; /* @@ -8222,6 +8182,27 @@ static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state) pfit_w * pfit_h); } +static void intel_mode_from_crtc_timings(struct drm_display_mode *mode, + const struct drm_display_mode *timings) +{ + mode->hdisplay = timings->crtc_hdisplay; + mode->htotal = timings->crtc_htotal; + mode->hsync_start = timings->crtc_hsync_start; + mode->hsync_end = timings->crtc_hsync_end; + + mode->vdisplay = timings->crtc_vdisplay; + mode->vtotal = timings->crtc_vtotal; + mode->vsync_start = timings->crtc_vsync_start; + mode->vsync_end = timings->crtc_vsync_end; + + mode->flags = timings->flags; + mode->type = DRM_MODE_TYPE_DRIVER; + + mode->clock = timings->crtc_clock; + + drm_mode_set_name(mode); +} + static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); @@ -8229,19 +8210,75 @@ static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state) if (HAS_GMCH(dev_priv)) /* FIXME calculate proper pipe pixel rate for GMCH pfit */ crtc_state->pixel_rate = - crtc_state->hw.adjusted_mode.crtc_clock; + crtc_state->hw.pipe_mode.crtc_clock; else crtc_state->pixel_rate = ilk_pipe_pixel_rate(crtc_state); } +static void intel_crtc_readout_derived_state(struct intel_crtc_state *crtc_state) +{ + struct drm_display_mode *mode = &crtc_state->hw.mode; + struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; + struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; + + drm_mode_copy(pipe_mode, adjusted_mode); + + if (crtc_state->bigjoiner) { + /* + * transcoder is programmed to the full mode, + * but pipe timings are half of the transcoder mode + */ + pipe_mode->crtc_hdisplay /= 2; + pipe_mode->crtc_hblank_start /= 2; + pipe_mode->crtc_hblank_end /= 2; + pipe_mode->crtc_hsync_start /= 2; + pipe_mode->crtc_hsync_end /= 2; + pipe_mode->crtc_htotal /= 2; + pipe_mode->crtc_clock /= 2; + } + + intel_mode_from_crtc_timings(pipe_mode, pipe_mode); + intel_mode_from_crtc_timings(adjusted_mode, adjusted_mode); + + intel_crtc_compute_pixel_rate(crtc_state); + + drm_mode_copy(mode, adjusted_mode); + mode->hdisplay = crtc_state->pipe_src_w << crtc_state->bigjoiner; + mode->vdisplay = crtc_state->pipe_src_h; +} + +static void intel_encoder_get_config(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state) +{ + encoder->get_config(encoder, crtc_state); + + intel_crtc_readout_derived_state(crtc_state); +} + static int intel_crtc_compute_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + struct drm_display_mode *pipe_mode = &pipe_config->hw.pipe_mode; int clock_limit = dev_priv->max_dotclk_freq; + drm_mode_copy(pipe_mode, &pipe_config->hw.adjusted_mode); + + /* Adjust pipe_mode for bigjoiner, with half the horizontal mode */ + if (pipe_config->bigjoiner) { + pipe_mode->crtc_clock /= 2; + pipe_mode->crtc_hdisplay /= 2; + pipe_mode->crtc_hblank_start /= 2; + pipe_mode->crtc_hblank_end /= 2; + pipe_mode->crtc_hsync_start /= 2; + pipe_mode->crtc_hsync_end /= 2; + pipe_mode->crtc_htotal /= 2; + pipe_config->pipe_src_w /= 2; + } + + intel_mode_from_crtc_timings(pipe_mode, pipe_mode); + if (INTEL_GEN(dev_priv) < 4) { clock_limit = dev_priv->max_cdclk_freq * 9 / 10; @@ -8250,16 +8287,16 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, * is > 90% of the (display) core speed. */ if (intel_crtc_supports_double_wide(crtc) && - adjusted_mode->crtc_clock > clock_limit) { + pipe_mode->crtc_clock > clock_limit) { clock_limit = dev_priv->max_dotclk_freq; pipe_config->double_wide = true; } } - if (adjusted_mode->crtc_clock > clock_limit) { + if (pipe_mode->crtc_clock > clock_limit) { drm_dbg_kms(&dev_priv->drm, "requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n", - adjusted_mode->crtc_clock, clock_limit, + pipe_mode->crtc_clock, clock_limit, yesno(pipe_config->double_wide)); return -EINVAL; } @@ -8302,7 +8339,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw. */ if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) && - adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay) + pipe_mode->crtc_hsync_start == pipe_mode->crtc_hdisplay) return -EINVAL; intel_crtc_compute_pixel_rate(pipe_config); @@ -9113,30 +9150,6 @@ static void intel_get_pipe_src_size(struct intel_crtc *crtc, tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe)); pipe_config->pipe_src_h = (tmp & 0xffff) + 1; pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1; - - pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h; - pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w; -} - -void intel_mode_from_pipe_config(struct drm_display_mode *mode, - struct intel_crtc_state *pipe_config) -{ - mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay; - mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal; - mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start; - mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end; - - mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay; - mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal; - mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start; - mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end; - - mode->flags = pipe_config->hw.adjusted_mode.flags; - mode->type = DRM_MODE_TYPE_DRIVER; - - mode->clock = pipe_config->hw.adjusted_mode.crtc_clock; - - drm_mode_set_name(mode); } static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state) @@ -10703,6 +10716,7 @@ static void skl_get_initial_plane_config(struct intel_crtc *crtc, struct intel_initial_plane_config *plane_config) { + struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *plane = to_intel_plane(crtc->base.primary); @@ -10719,6 +10733,12 @@ skl_get_initial_plane_config(struct intel_crtc *crtc, drm_WARN_ON(dev, pipe != crtc->pipe); + if (crtc_state->bigjoiner) { + drm_dbg_kms(&dev_priv->drm, + "Unsupported bigjoiner configuration for initial FB\n"); + return; + } + intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); if (!intel_fb) { drm_dbg_kms(&dev_priv->drm, "failed to alloc fb\n"); @@ -10927,6 +10947,7 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, if (intel_de_read(dev_priv, PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) { struct intel_shared_dpll *pll; enum intel_dpll_id pll_id; + bool pll_active; pipe_config->has_pch_encoder = true; @@ -10954,8 +10975,9 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, intel_get_shared_dpll_by_id(dev_priv, pll_id); pll = pipe_config->shared_dpll; - drm_WARN_ON(dev, !pll->info->funcs->get_hw_state(dev_priv, pll, - &pipe_config->dpll_hw_state)); + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state); + drm_WARN_ON(dev, !pll_active); tmp = pipe_config->dpll_hw_state.dpll; pipe_config->pixel_multiplier = @@ -11003,19 +11025,32 @@ static int hsw_crtc_compute_clock(struct intel_crtc *crtc, return 0; } -static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, +static void dg1_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, struct intel_crtc_state *pipe_config) { + enum icl_port_dpll_id port_dpll_id = ICL_PORT_DPLL_DEFAULT; + enum phy phy = intel_port_to_phy(dev_priv, port); + struct icl_port_dpll *port_dpll; + struct intel_shared_dpll *pll; enum intel_dpll_id id; - u32 temp; + bool pll_active; + u32 clk_sel; - temp = intel_de_read(dev_priv, DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); - id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); + clk_sel = intel_de_read(dev_priv, DG1_DPCLKA_CFGCR0(phy)) & DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy); + id = DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_DPLL_MAP(clk_sel, phy); - if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL2)) + if (WARN_ON(id > DPLL_ID_DG1_DPLL3)) return; - pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); + pll = intel_get_shared_dpll_by_id(dev_priv, id); + port_dpll = &pipe_config->icl_port_dplls[port_dpll_id]; + + port_dpll->pll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &port_dpll->hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); + + icl_set_active_port_dpll(pipe_config, port_dpll_id); } static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, @@ -11023,7 +11058,10 @@ static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, { enum phy phy = intel_port_to_phy(dev_priv, port); enum icl_port_dpll_id port_dpll_id; + struct icl_port_dpll *port_dpll; + struct intel_shared_dpll *pll; enum intel_dpll_id id; + bool pll_active; u32 temp; if (intel_phy_is_combo(dev_priv, phy)) { @@ -11058,17 +11096,46 @@ static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, return; } - pipe_config->icl_port_dplls[port_dpll_id].pll = - intel_get_shared_dpll_by_id(dev_priv, id); + pll = intel_get_shared_dpll_by_id(dev_priv, id); + port_dpll = &pipe_config->icl_port_dplls[port_dpll_id]; + + port_dpll->pll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &port_dpll->hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); icl_set_active_port_dpll(pipe_config, port_dpll_id); } +static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, + struct intel_crtc_state *pipe_config) +{ + struct intel_shared_dpll *pll; + enum intel_dpll_id id; + bool pll_active; + u32 temp; + + temp = intel_de_read(dev_priv, DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); + id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); + + if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL2)) + return; + + pll = intel_get_shared_dpll_by_id(dev_priv, id); + + pipe_config->shared_dpll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); +} + static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, struct intel_crtc_state *pipe_config) { + struct intel_shared_dpll *pll; enum intel_dpll_id id; + bool pll_active; switch (port) { case PORT_A: @@ -11085,13 +11152,20 @@ static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv, return; } - pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); + pll = intel_get_shared_dpll_by_id(dev_priv, id); + + pipe_config->shared_dpll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); } static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, struct intel_crtc_state *pipe_config) { + struct intel_shared_dpll *pll; enum intel_dpll_id id; + bool pll_active; u32 temp; temp = intel_de_read(dev_priv, DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port); @@ -11100,14 +11174,21 @@ static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL3)) return; - pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); + pll = intel_get_shared_dpll_by_id(dev_priv, id); + + pipe_config->shared_dpll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); } static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, struct intel_crtc_state *pipe_config) { + struct intel_shared_dpll *pll; enum intel_dpll_id id; u32 ddi_pll_sel = intel_de_read(dev_priv, PORT_CLK_SEL(port)); + bool pll_active; switch (ddi_pll_sel) { case PORT_CLK_SEL_WRPLL1: @@ -11135,7 +11216,12 @@ static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, return; } - pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); + pll = intel_get_shared_dpll_by_id(dev_priv, id); + + pipe_config->shared_dpll = pll; + pll_active = intel_dpll_get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state); + drm_WARN_ON(&dev_priv->drm, !pll_active); } static bool hsw_get_transcoder_state(struct intel_crtc *crtc, @@ -11295,7 +11381,6 @@ static void hsw_get_ddi_port_state(struct intel_crtc *crtc, { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; - struct intel_shared_dpll *pll; enum port port; u32 tmp; @@ -11305,30 +11390,27 @@ static void hsw_get_ddi_port_state(struct intel_crtc *crtc, } else { tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); + if (!(tmp & TRANS_DDI_FUNC_ENABLE)) + return; if (INTEL_GEN(dev_priv) >= 12) port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp); else port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp); } - if (INTEL_GEN(dev_priv) >= 11) + if (IS_DG1(dev_priv)) + dg1_get_ddi_pll(dev_priv, port, pipe_config); + else if (INTEL_GEN(dev_priv) >= 11) icl_get_ddi_pll(dev_priv, port, pipe_config); else if (IS_CANNONLAKE(dev_priv)) cnl_get_ddi_pll(dev_priv, port, pipe_config); - else if (IS_GEN9_BC(dev_priv)) - skl_get_ddi_pll(dev_priv, port, pipe_config); else if (IS_GEN9_LP(dev_priv)) bxt_get_ddi_pll(dev_priv, port, pipe_config); + else if (IS_GEN9_BC(dev_priv)) + skl_get_ddi_pll(dev_priv, port, pipe_config); else hsw_get_ddi_pll(dev_priv, port, pipe_config); - pll = pipe_config->shared_dpll; - if (pll) { - drm_WARN_ON(&dev_priv->drm, - !pll->info->funcs->get_hw_state(dev_priv, pll, - &pipe_config->dpll_hw_state)); - } - /* * Haswell has only FDI/PCH transcoder A. It is which is connected to * DDI E. So just check whether this pipe is wired to DDI E and whether @@ -11378,10 +11460,19 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, active = true; } - if (!active) - goto out; + intel_dsc_get_config(pipe_config); + + if (!active) { + /* bigjoiner slave doesn't enable transcoder */ + if (!pipe_config->bigjoiner_slave) + goto out; - if (!transcoder_is_dsi(pipe_config->cpu_transcoder) || + active = true; + pipe_config->pixel_multiplier = 1; + + /* we cannot read out most state, so don't bother.. */ + pipe_config->quirks |= PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE; + } else if (!transcoder_is_dsi(pipe_config->cpu_transcoder) || INTEL_GEN(dev_priv) >= 11) { hsw_get_ddi_port_state(crtc, pipe_config); intel_get_transcoder_timings(crtc, pipe_config); @@ -11456,7 +11547,10 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, } } - if (pipe_config->cpu_transcoder != TRANSCODER_EDP && + if (pipe_config->bigjoiner_slave) { + /* Cannot be read out as a slave, set to 0. */ + pipe_config->pixel_multiplier = 0; + } else if (pipe_config->cpu_transcoder != TRANSCODER_EDP && !transcoder_is_dsi(pipe_config->cpu_transcoder)) { pipe_config->pixel_multiplier = intel_de_read(dev_priv, @@ -11473,6 +11567,21 @@ out: return active; } +static bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + if (!i915->display.get_pipe_config(crtc, crtc_state)) + return false; + + crtc_state->hw.active = true; + + intel_crtc_readout_derived_state(crtc_state); + + return true; +} + static u32 intel_cursor_base(const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = @@ -11578,6 +11687,8 @@ static int intel_check_cursor(struct intel_crtc_state *crtc_state, { const struct drm_framebuffer *fb = plane_state->hw.fb; struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev); + const struct drm_rect src = plane_state->uapi.src; + const struct drm_rect dst = plane_state->uapi.dst; int ret; if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) { @@ -11585,17 +11696,16 @@ static int intel_check_cursor(struct intel_crtc_state *crtc_state, return -EINVAL; } - ret = drm_atomic_helper_check_plane_state(&plane_state->uapi, - &crtc_state->uapi, - DRM_PLANE_HELPER_NO_SCALING, - DRM_PLANE_HELPER_NO_SCALING, - true, true); + ret = intel_atomic_plane_check_clipping(plane_state, crtc_state, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + true); if (ret) return ret; /* Use the unclipped src/dst rectangles, which we program to hw */ - plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi); - plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi); + plane_state->uapi.src = src; + plane_state->uapi.dst = dst; ret = intel_cursor_check_surface(plane_state); if (ret) @@ -12469,15 +12579,15 @@ intel_encoder_current_mode(struct intel_encoder *encoder) return NULL; } - if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) { + if (!intel_crtc_get_pipe_config(crtc_state)) { kfree(crtc_state); kfree(mode); return NULL; } - encoder->get_config(encoder, crtc_state); + intel_encoder_get_config(encoder, crtc_state); - intel_mode_from_pipe_config(mode, crtc_state); + intel_mode_from_crtc_timings(mode, &crtc_state->hw.adjusted_mode); kfree(crtc_state); @@ -12663,7 +12773,7 @@ static bool encoders_cloneable(const struct intel_encoder *a, b->cloneable & (1 << a->type)); } -static bool check_single_encoder_cloning(struct drm_atomic_state *state, +static bool check_single_encoder_cloning(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) { @@ -12672,7 +12782,7 @@ static bool check_single_encoder_cloning(struct drm_atomic_state *state, struct drm_connector_state *connector_state; int i; - for_each_new_connector_in_state(state, connector, connector_state, i) { + for_each_new_connector_in_state(&state->base, connector, connector_state, i) { if (connector_state->crtc != &crtc->base) continue; @@ -12787,7 +12897,7 @@ static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state) memcpy(linked_state->color_plane, plane_state->color_plane, sizeof(linked_state->color_plane)); - intel_plane_copy_uapi_to_hw_state(linked_state, plane_state); + intel_plane_copy_hw_state(linked_state, plane_state); linked_state->uapi.src = plane_state->uapi.src; linked_state->uapi.dst = plane_state->uapi.dst; @@ -12821,15 +12931,15 @@ static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state) static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state) { - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; int linetime_wm; if (!crtc_state->hw.enable) return 0; - linetime_wm = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, - adjusted_mode->crtc_clock); + linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8, + pipe_mode->crtc_clock); return min(linetime_wm, 0x1ff); } @@ -12837,14 +12947,14 @@ static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state) static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state, const struct intel_cdclk_state *cdclk_state) { - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; int linetime_wm; if (!crtc_state->hw.enable) return 0; - linetime_wm = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, + linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8, cdclk_state->logical.cdclk); return min(linetime_wm, 0x1ff); @@ -12854,14 +12964,14 @@ static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; int linetime_wm; if (!crtc_state->hw.enable) return 0; - linetime_wm = DIV_ROUND_UP(adjusted_mode->crtc_htotal * 1000 * 8, + linetime_wm = DIV_ROUND_UP(pipe_mode->crtc_htotal * 1000 * 8, crtc_state->pixel_rate); /* Display WA #1135: BXT:ALL GLK:ALL */ @@ -12912,6 +13022,7 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, if (mode_changed && crtc_state->hw.enable && dev_priv->display.crtc_compute_clock && + !crtc_state->bigjoiner_slave && !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) { ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state); if (ret) @@ -13038,10 +13149,11 @@ compute_sink_pipe_bpp(const struct drm_connector_state *conn_state, case 10 ... 11: bpp = 10 * 3; break; - case 12: + case 12 ... 16: bpp = 12 * 3; break; default: + MISSING_CASE(conn_state->max_bpc); return -EINVAL; } @@ -13218,11 +13330,11 @@ static void intel_dump_plane_state(const struct intel_plane_state *plane_state) } drm_dbg_kms(&i915->drm, - "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n", + "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s modifier = 0x%llx, visible: %s\n", plane->base.base.id, plane->base.name, fb->base.id, fb->width, fb->height, drm_get_format_name(fb->format->format, &format_name), - yesno(plane_state->uapi.visible)); + fb->modifier, yesno(plane_state->uapi.visible)); drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n", plane_state->hw.rotation, plane_state->scaler_id); if (plane_state->uapi.visible) @@ -13270,6 +13382,10 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, transcoder_name(pipe_config->master_transcoder), pipe_config->sync_mode_slaves_mask); + drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s\n", + pipe_config->bigjoiner_slave ? "slave" : + pipe_config->bigjoiner ? "master" : "no"); + if (pipe_config->has_pch_encoder) intel_dump_m_n_config(pipe_config, "fdi", pipe_config->fdi_lanes, @@ -13317,6 +13433,9 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n"); drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode); intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode); + drm_dbg_kms(&dev_priv->drm, "pipe mode:\n"); + drm_mode_debug_printmodeline(&pipe_config->hw.pipe_mode); + intel_dump_crtc_timings(dev_priv, &pipe_config->hw.pipe_mode); drm_dbg_kms(&dev_priv->drm, "port clock: %d, pipe src size: %dx%d, pixel rate %d\n", pipe_config->port_clock, @@ -13448,24 +13567,41 @@ static bool check_digital_port_conflicts(struct intel_atomic_state *state) } static void -intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state) +intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state) { - intel_crtc_copy_color_blobs(crtc_state); + const struct intel_crtc_state *from_crtc_state = crtc_state; + + if (crtc_state->bigjoiner_slave) { + from_crtc_state = intel_atomic_get_new_crtc_state(state, + crtc_state->bigjoiner_linked_crtc); + + /* No need to copy state if the master state is unchanged */ + if (!from_crtc_state) + return; + } + + intel_crtc_copy_color_blobs(crtc_state, from_crtc_state); } static void -intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state) +intel_crtc_copy_uapi_to_hw_state(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state) { crtc_state->hw.enable = crtc_state->uapi.enable; crtc_state->hw.active = crtc_state->uapi.active; crtc_state->hw.mode = crtc_state->uapi.mode; crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode; crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter; - intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state); + + intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc_state); } static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state) { + if (crtc_state->bigjoiner_slave) + return; + crtc_state->uapi.enable = crtc_state->hw.enable; crtc_state->uapi.active = crtc_state->hw.active; drm_WARN_ON(crtc_state->uapi.crtc->dev, @@ -13484,7 +13620,49 @@ static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state } static int -intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state) +copy_bigjoiner_crtc_state(struct intel_crtc_state *crtc_state, + const struct intel_crtc_state *from_crtc_state) +{ + struct intel_crtc_state *saved_state; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + saved_state = kmemdup(from_crtc_state, sizeof(*saved_state), GFP_KERNEL); + if (!saved_state) + return -ENOMEM; + + saved_state->uapi = crtc_state->uapi; + saved_state->scaler_state = crtc_state->scaler_state; + saved_state->shared_dpll = crtc_state->shared_dpll; + saved_state->dpll_hw_state = crtc_state->dpll_hw_state; + saved_state->crc_enabled = crtc_state->crc_enabled; + + intel_crtc_free_hw_state(crtc_state); + memcpy(crtc_state, saved_state, sizeof(*crtc_state)); + kfree(saved_state); + + /* Re-init hw state */ + memset(&crtc_state->hw, 0, sizeof(saved_state->hw)); + crtc_state->hw.enable = from_crtc_state->hw.enable; + crtc_state->hw.active = from_crtc_state->hw.active; + crtc_state->hw.pipe_mode = from_crtc_state->hw.pipe_mode; + crtc_state->hw.adjusted_mode = from_crtc_state->hw.adjusted_mode; + + /* Some fixups */ + crtc_state->uapi.mode_changed = from_crtc_state->uapi.mode_changed; + crtc_state->uapi.connectors_changed = from_crtc_state->uapi.connectors_changed; + crtc_state->uapi.active_changed = from_crtc_state->uapi.active_changed; + crtc_state->nv12_planes = crtc_state->c8_planes = crtc_state->update_planes = 0; + crtc_state->bigjoiner_linked_crtc = to_intel_crtc(from_crtc_state->uapi.crtc); + crtc_state->bigjoiner_slave = true; + crtc_state->cpu_transcoder = (enum transcoder)crtc->pipe; + crtc_state->has_audio = false; + + return 0; +} + +static int +intel_crtc_prepare_cleared_state(struct intel_atomic_state *state, + struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -13516,16 +13694,16 @@ intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state) memcpy(crtc_state, saved_state, sizeof(*crtc_state)); kfree(saved_state); - intel_crtc_copy_uapi_to_hw_state(crtc_state); + intel_crtc_copy_uapi_to_hw_state(state, crtc_state); return 0; } static int -intel_modeset_pipe_config(struct intel_crtc_state *pipe_config) +intel_modeset_pipe_config(struct intel_atomic_state *state, + struct intel_crtc_state *pipe_config) { struct drm_crtc *crtc = pipe_config->uapi.crtc; - struct drm_atomic_state *state = pipe_config->uapi.state; struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev); struct drm_connector *connector; struct drm_connector_state *connector_state; @@ -13567,7 +13745,7 @@ intel_modeset_pipe_config(struct intel_crtc_state *pipe_config) &pipe_config->pipe_src_w, &pipe_config->pipe_src_h); - for_each_new_connector_in_state(state, connector, connector_state, i) { + for_each_new_connector_in_state(&state->base, connector, connector_state, i) { struct intel_encoder *encoder = to_intel_encoder(connector_state->best_encoder); @@ -13605,7 +13783,7 @@ encoder_retry: * adjust it according to limitations or connector properties, and also * a chance to reject the mode entirely. */ - for_each_new_connector_in_state(state, connector, connector_state, i) { + for_each_new_connector_in_state(&state->base, connector, connector_state, i) { struct intel_encoder *encoder = to_intel_encoder(connector_state->best_encoder); @@ -14065,21 +14243,53 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_X(output_types); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end); - - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start); - PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end); - - PIPE_CONF_CHECK_I(pixel_multiplier); + /* FIXME do the readout properly and get rid of this quirk */ + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) { + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hdisplay); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_htotal); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_start); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_end); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_start); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_end); + + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vdisplay); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vtotal); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_start); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_end); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_start); + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_end); + + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end); + + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end); + + PIPE_CONF_CHECK_I(pixel_multiplier); + + PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, + DRM_MODE_FLAG_INTERLACE); + + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) { + PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, + DRM_MODE_FLAG_PHSYNC); + PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, + DRM_MODE_FLAG_NHSYNC); + PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, + DRM_MODE_FLAG_PVSYNC); + PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, + DRM_MODE_FLAG_NVSYNC); + } + } + PIPE_CONF_CHECK_I(output_format); PIPE_CONF_CHECK_BOOL(has_hdmi_sink); if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) || @@ -14089,24 +14299,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_BOOL(hdmi_scrambling); PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio); PIPE_CONF_CHECK_BOOL(has_infoframe); - PIPE_CONF_CHECK_BOOL(fec_enable); + /* FIXME do the readout properly and get rid of this quirk */ + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) + PIPE_CONF_CHECK_BOOL(fec_enable); PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio); - PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, - DRM_MODE_FLAG_INTERLACE); - - if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) { - PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, - DRM_MODE_FLAG_PHSYNC); - PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, - DRM_MODE_FLAG_NHSYNC); - PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, - DRM_MODE_FLAG_PVSYNC); - PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags, - DRM_MODE_FLAG_NVSYNC); - } - PIPE_CONF_CHECK_X(gmch_pfit.control); /* pfit ratios are autocomputed by the hw on gen4+ */ if (INTEL_GEN(dev_priv) < 4) @@ -14132,7 +14330,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, } PIPE_CONF_CHECK_I(scaler_state.scaler_id); - PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate); + /* FIXME do the readout properly and get rid of this quirk */ + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) + PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate); PIPE_CONF_CHECK_X(gamma_mode); if (IS_CHERRYVIEW(dev_priv)) @@ -14153,48 +14353,53 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_BOOL(double_wide); PIPE_CONF_CHECK_P(shared_dpll); - PIPE_CONF_CHECK_X(dpll_hw_state.dpll); - PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md); - PIPE_CONF_CHECK_X(dpll_hw_state.fp0); - PIPE_CONF_CHECK_X(dpll_hw_state.fp1); - PIPE_CONF_CHECK_X(dpll_hw_state.wrpll); - PIPE_CONF_CHECK_X(dpll_hw_state.spll); - PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1); - PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1); - PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2); - PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0); - PIPE_CONF_CHECK_X(dpll_hw_state.ebb0); - PIPE_CONF_CHECK_X(dpll_hw_state.ebb4); - PIPE_CONF_CHECK_X(dpll_hw_state.pll0); - PIPE_CONF_CHECK_X(dpll_hw_state.pll1); - PIPE_CONF_CHECK_X(dpll_hw_state.pll2); - PIPE_CONF_CHECK_X(dpll_hw_state.pll3); - PIPE_CONF_CHECK_X(dpll_hw_state.pll6); - PIPE_CONF_CHECK_X(dpll_hw_state.pll8); - PIPE_CONF_CHECK_X(dpll_hw_state.pll9); - PIPE_CONF_CHECK_X(dpll_hw_state.pll10); - PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias); - PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias); - - PIPE_CONF_CHECK_X(dsi_pll.ctrl); - PIPE_CONF_CHECK_X(dsi_pll.div); - - if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) - PIPE_CONF_CHECK_I(pipe_bpp); - - PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock); - PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); - - PIPE_CONF_CHECK_I(min_voltage_level); + + /* FIXME do the readout properly and get rid of this quirk */ + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) { + PIPE_CONF_CHECK_X(dpll_hw_state.dpll); + PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md); + PIPE_CONF_CHECK_X(dpll_hw_state.fp0); + PIPE_CONF_CHECK_X(dpll_hw_state.fp1); + PIPE_CONF_CHECK_X(dpll_hw_state.wrpll); + PIPE_CONF_CHECK_X(dpll_hw_state.spll); + PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1); + PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1); + PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2); + PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0); + PIPE_CONF_CHECK_X(dpll_hw_state.ebb0); + PIPE_CONF_CHECK_X(dpll_hw_state.ebb4); + PIPE_CONF_CHECK_X(dpll_hw_state.pll0); + PIPE_CONF_CHECK_X(dpll_hw_state.pll1); + PIPE_CONF_CHECK_X(dpll_hw_state.pll2); + PIPE_CONF_CHECK_X(dpll_hw_state.pll3); + PIPE_CONF_CHECK_X(dpll_hw_state.pll6); + PIPE_CONF_CHECK_X(dpll_hw_state.pll8); + PIPE_CONF_CHECK_X(dpll_hw_state.pll9); + PIPE_CONF_CHECK_X(dpll_hw_state.pll10); + PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias); + PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias); + + PIPE_CONF_CHECK_X(dsi_pll.ctrl); + PIPE_CONF_CHECK_X(dsi_pll.div); + + if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) + PIPE_CONF_CHECK_I(pipe_bpp); + + PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock); + PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock); + PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); + + PIPE_CONF_CHECK_I(min_voltage_level); + } PIPE_CONF_CHECK_X(infoframes.enable); PIPE_CONF_CHECK_X(infoframes.gcp); @@ -14206,6 +14411,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_X(sync_mode_slaves_mask); PIPE_CONF_CHECK_I(master_transcoder); + PIPE_CONF_CHECK_BOOL(bigjoiner); + PIPE_CONF_CHECK_BOOL(bigjoiner_slave); + PIPE_CONF_CHECK_P(bigjoiner_linked_crtc); PIPE_CONF_CHECK_I(dsc.compression_enable); PIPE_CONF_CHECK_I(dsc.dsc_split); @@ -14477,6 +14685,7 @@ verify_crtc_state(struct intel_crtc *crtc, struct intel_encoder *encoder; struct intel_crtc_state *pipe_config = old_crtc_state; struct drm_atomic_state *state = old_crtc_state->uapi.state; + struct intel_crtc *master = crtc; __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi); intel_crtc_free_hw_state(old_crtc_state); @@ -14488,8 +14697,7 @@ verify_crtc_state(struct intel_crtc *crtc, pipe_config->hw.enable = new_crtc_state->hw.enable; - pipe_config->hw.active = - dev_priv->display.get_pipe_config(crtc, pipe_config); + intel_crtc_get_pipe_config(pipe_config); /* we keep both pipes enabled on 830 */ if (IS_I830(dev_priv) && pipe_config->hw.active) @@ -14505,7 +14713,10 @@ verify_crtc_state(struct intel_crtc *crtc, "(expected %i, found %i)\n", new_crtc_state->hw.active, crtc->active); - for_each_encoder_on_crtc(dev, &crtc->base, encoder) { + if (new_crtc_state->bigjoiner_slave) + master = new_crtc_state->bigjoiner_linked_crtc; + + for_each_encoder_on_crtc(dev, &master->base, encoder) { enum pipe pipe; bool active; @@ -14515,16 +14726,14 @@ verify_crtc_state(struct intel_crtc *crtc, encoder->base.base.id, active, new_crtc_state->hw.active); - I915_STATE_WARN(active && crtc->pipe != pipe, + I915_STATE_WARN(active && master->pipe != pipe, "Encoder connected to wrong pipe %c\n", pipe_name(pipe)); if (active) - encoder->get_config(encoder, pipe_config); + intel_encoder_get_config(encoder, pipe_config); } - intel_crtc_compute_pixel_rate(pipe_config); - if (!new_crtc_state->hw.active) return; @@ -14565,7 +14774,7 @@ verify_single_dpll_state(struct drm_i915_private *dev_priv, drm_dbg_kms(&dev_priv->drm, "%s\n", pll->info->name); - active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state); + active = intel_dpll_get_hw_state(dev_priv, pll, &dpll_hw_state); if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) { I915_STATE_WARN(!pll->on && pll->active_mask, @@ -14901,6 +15110,44 @@ static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv) IS_IVYBRIDGE(dev_priv) || (INTEL_GEN(dev_priv) >= 11); } +static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct intel_crtc *other) +{ + const struct intel_plane_state *plane_state; + struct intel_plane *plane; + u8 plane_ids = 0; + int i; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe == crtc->pipe) + plane_ids |= BIT(plane->id); + } + + return intel_crtc_add_planes_to_state(state, other, plane_ids); +} + +static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state) +{ + const struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + int i; + + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { + int ret; + + if (!crtc_state->bigjoiner) + continue; + + ret = intel_crtc_add_bigjoiner_planes(state, crtc, + crtc_state->bigjoiner_linked_crtc); + if (ret) + return ret; + } + + return 0; +} + static int intel_atomic_check_planes(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); @@ -14914,6 +15161,10 @@ static int intel_atomic_check_planes(struct intel_atomic_state *state) if (ret) return ret; + ret = intel_bigjoiner_add_affected_planes(state); + if (ret) + return ret; + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { ret = intel_plane_atomic_check(state, plane); if (ret) { @@ -15052,6 +15303,75 @@ static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state, return false; } +static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_crtc_state *slave_crtc_state, *master_crtc_state; + struct intel_crtc *slave, *master; + + /* slave being enabled, is master is still claiming this crtc? */ + if (old_crtc_state->bigjoiner_slave) { + slave = crtc; + master = old_crtc_state->bigjoiner_linked_crtc; + master_crtc_state = intel_atomic_get_new_crtc_state(state, master); + if (!master_crtc_state || !needs_modeset(master_crtc_state)) + goto claimed; + } + + if (!new_crtc_state->bigjoiner) + return 0; + + if (1 + crtc->pipe >= INTEL_NUM_PIPES(dev_priv)) { + DRM_DEBUG_KMS("[CRTC:%d:%s] Big joiner configuration requires " + "CRTC + 1 to be used, doesn't exist\n", + crtc->base.base.id, crtc->base.name); + return -EINVAL; + } + + slave = new_crtc_state->bigjoiner_linked_crtc = + intel_get_crtc_for_pipe(dev_priv, crtc->pipe + 1); + slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave); + master = crtc; + if (IS_ERR(slave_crtc_state)) + return PTR_ERR(slave_crtc_state); + + /* master being enabled, slave was already configured? */ + if (slave_crtc_state->uapi.enable) + goto claimed; + + DRM_DEBUG_KMS("[CRTC:%d:%s] Used as slave for big joiner\n", + slave->base.base.id, slave->base.name); + + return copy_bigjoiner_crtc_state(slave_crtc_state, new_crtc_state); + +claimed: + DRM_DEBUG_KMS("[CRTC:%d:%s] Slave is enabled as normal CRTC, but " + "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n", + slave->base.base.id, slave->base.name, + master->base.base.id, master->base.name); + return -EINVAL; +} + +static int kill_bigjoiner_slave(struct intel_atomic_state *state, + struct intel_crtc_state *master_crtc_state) +{ + struct intel_crtc_state *slave_crtc_state = + intel_atomic_get_crtc_state(&state->base, + master_crtc_state->bigjoiner_linked_crtc); + + if (IS_ERR(slave_crtc_state)) + return PTR_ERR(slave_crtc_state); + + slave_crtc_state->bigjoiner = master_crtc_state->bigjoiner = false; + slave_crtc_state->bigjoiner_slave = master_crtc_state->bigjoiner_slave = false; + slave_crtc_state->bigjoiner_linked_crtc = master_crtc_state->bigjoiner_linked_crtc = NULL; + intel_crtc_copy_uapi_to_hw_state(state, slave_crtc_state); + return 0; +} + /** * DOC: asynchronous flip implementation * @@ -15185,6 +15505,27 @@ static int intel_atomic_check_async(struct intel_atomic_state *state) return 0; } +static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state) +{ + const struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + int i; + + for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) { + struct intel_crtc_state *linked_crtc_state; + + if (!crtc_state->bigjoiner) + continue; + + linked_crtc_state = intel_atomic_get_crtc_state(&state->base, + crtc_state->bigjoiner_linked_crtc); + if (IS_ERR(linked_crtc_state)) + return PTR_ERR(linked_crtc_state); + } + + return 0; +} + /** * intel_atomic_check - validate state object * @dev: drm device @@ -15210,23 +15551,44 @@ static int intel_atomic_check(struct drm_device *dev, if (ret) goto fail; + ret = intel_bigjoiner_add_affected_crtcs(state); + if (ret) + goto fail; + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (!needs_modeset(new_crtc_state)) { /* Light copy */ - intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state); + intel_crtc_copy_uapi_to_hw_state_nomodeset(state, new_crtc_state); continue; } - ret = intel_crtc_prepare_cleared_state(new_crtc_state); + /* Kill old bigjoiner link, we may re-establish afterwards */ + if (old_crtc_state->bigjoiner && !old_crtc_state->bigjoiner_slave) { + ret = kill_bigjoiner_slave(state, new_crtc_state); + if (ret) + goto fail; + } + + if (!new_crtc_state->uapi.enable) { + if (!new_crtc_state->bigjoiner_slave) { + intel_crtc_copy_uapi_to_hw_state(state, new_crtc_state); + any_ms = true; + } + continue; + } + + ret = intel_crtc_prepare_cleared_state(state, new_crtc_state); if (ret) goto fail; - if (!new_crtc_state->hw.enable) - continue; + ret = intel_modeset_pipe_config(state, new_crtc_state); + if (ret) + goto fail; - ret = intel_modeset_pipe_config(new_crtc_state); + ret = intel_atomic_check_bigjoiner(state, crtc, old_crtc_state, + new_crtc_state); if (ret) goto fail; } @@ -15528,6 +15890,9 @@ static void intel_enable_crtc(struct intel_atomic_state *state, dev_priv->display.crtc_enable(state, crtc); + if (new_crtc_state->bigjoiner_slave) + return; + /* vblanks work again, re-enable pipe CRC. */ intel_crtc_enable_pipe_crc(crtc); } @@ -15582,7 +15947,6 @@ static void intel_update_crtc(struct intel_atomic_state *state, intel_crtc_arm_fifo_underrun(crtc, new_crtc_state); } - static void intel_old_crtc_state_disables(struct intel_atomic_state *state, struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *new_crtc_state, @@ -15590,9 +15954,22 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state, { struct drm_i915_private *dev_priv = to_i915(state->base.dev); + drm_WARN_ON(&dev_priv->drm, old_crtc_state->bigjoiner_slave); + intel_crtc_disable_planes(state, crtc); /* + * We still need special handling for disabling bigjoiner master + * and slaves since for slave we do not have encoder or plls + * so we dont need to disable those. + */ + if (old_crtc_state->bigjoiner) { + intel_crtc_disable_planes(state, + old_crtc_state->bigjoiner_linked_crtc); + old_crtc_state->bigjoiner_linked_crtc->active = false; + } + + /* * We need to disable pipe CRC before disabling the pipe, * or we race against vblank off. */ @@ -15620,7 +15997,7 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state) /* Only disable port sync and MST slaves */ for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (!needs_modeset(new_crtc_state)) + if (!needs_modeset(new_crtc_state) || old_crtc_state->bigjoiner) continue; if (!old_crtc_state->hw.active) @@ -15645,10 +16022,18 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state) for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (!needs_modeset(new_crtc_state) || - (handled & BIT(crtc->pipe))) + (handled & BIT(crtc->pipe)) || + old_crtc_state->bigjoiner_slave) continue; intel_pre_plane_update(state, crtc); + if (old_crtc_state->bigjoiner) { + struct intel_crtc *slave = + old_crtc_state->bigjoiner_linked_crtc; + + intel_pre_plane_update(state, slave); + } + if (old_crtc_state->hw.active) intel_old_crtc_state_disables(state, old_crtc_state, new_crtc_state, crtc); @@ -15746,7 +16131,8 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) continue; if (intel_dp_mst_is_slave_trans(new_crtc_state) || - is_trans_port_sync_master(new_crtc_state)) + is_trans_port_sync_master(new_crtc_state) || + (new_crtc_state->bigjoiner && !new_crtc_state->bigjoiner_slave)) continue; modeset_pipes &= ~BIT(pipe); @@ -15756,7 +16142,7 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) /* * Then we enable all remaining pipes that depend on other - * pipes: MST slaves and port sync masters. + * pipes: MST slaves and port sync masters, big joiner master */ for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { enum pipe pipe = crtc->pipe; @@ -16548,9 +16934,11 @@ intel_legacy_cursor_update(struct drm_plane *_plane, /* * When crtc is inactive or there is a modeset pending, * wait for it to complete in the slowpath + * + * FIXME bigjoiner fastpath would be good */ if (!crtc_state->hw.active || needs_modeset(crtc_state) || - crtc_state->update_pipe) + crtc_state->update_pipe || crtc_state->bigjoiner) goto slow; /* @@ -16596,7 +16984,7 @@ intel_legacy_cursor_update(struct drm_plane *_plane, new_plane_state->uapi.crtc_w = crtc_w; new_plane_state->uapi.crtc_h = crtc_h; - intel_plane_copy_uapi_to_hw_state(new_plane_state, new_plane_state); + intel_plane_copy_uapi_to_hw_state(new_plane_state, new_plane_state, crtc); ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state, old_plane_state, new_plane_state); @@ -17213,7 +17601,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - if (IS_ROCKETLAKE(dev_priv)) { + if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) { intel_ddi_init(dev_priv, PORT_A); intel_ddi_init(dev_priv, PORT_B); intel_ddi_init(dev_priv, PORT_TC1); @@ -17223,7 +17611,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) intel_ddi_init(dev_priv, PORT_B); intel_ddi_init(dev_priv, PORT_TC1); intel_ddi_init(dev_priv, PORT_TC2); - intel_ddi_init(dev_priv, PORT_TC2); + intel_ddi_init(dev_priv, PORT_TC3); intel_ddi_init(dev_priv, PORT_TC4); intel_ddi_init(dev_priv, PORT_TC5); intel_ddi_init(dev_priv, PORT_TC6); @@ -17753,7 +18141,8 @@ intel_mode_valid(struct drm_device *dev, enum drm_mode_status intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv, - const struct drm_display_mode *mode) + const struct drm_display_mode *mode, + bool bigjoiner) { int plane_width_max, plane_height_max; @@ -17770,7 +18159,7 @@ intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv, * too big for that. */ if (INTEL_GEN(dev_priv) >= 11) { - plane_width_max = 5120; + plane_width_max = 5120 << bigjoiner; plane_height_max = 4320; } else { plane_width_max = 5120; @@ -18311,7 +18700,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915) for_each_intel_crtc(dev, crtc) { struct intel_initial_plane_config plane_config = {}; - if (!crtc->active) + if (!to_intel_crtc_state(crtc->base.state)->uapi.active) continue; /* @@ -18340,16 +18729,6 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915) if (!HAS_GMCH(i915)) sanitize_watermarks(i915); - /* - * Force all active planes to recompute their states. So that on - * mode_setcrtc after probe, all the intel_plane_state variables - * are already calculated and there is no assert_plane warnings - * during bootup. - */ - ret = intel_initial_commit(dev); - if (ret) - drm_dbg_kms(&i915->drm, "Initial commit in probe failed.\n"); - return 0; } @@ -18358,11 +18737,21 @@ int intel_modeset_init(struct drm_i915_private *i915) { int ret; - intel_overlay_setup(i915); - if (!HAS_DISPLAY(i915)) return 0; + /* + * Force all active planes to recompute their states. So that on + * mode_setcrtc after probe, all the intel_plane_state variables + * are already calculated and there is no assert_plane warnings + * during bootup. + */ + ret = intel_initial_commit(&i915->drm); + if (ret) + drm_dbg_kms(&i915->drm, "Initial modeset failed, %d\n", ret); + + intel_overlay_setup(i915); + ret = intel_fbdev_init(&i915->drm); if (ret) return ret; @@ -18373,8 +18762,6 @@ int intel_modeset_init(struct drm_i915_private *i915) intel_init_ipc(i915); - intel_psr_set_force_mode_changed(i915->psr.dp); - return 0; } @@ -18624,7 +19011,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc, /* Adjust the state of the output pipe according to whether we * have active connectors/encoders. */ - if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc)) + if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc) && + !crtc_state->bigjoiner_slave) intel_crtc_disable_noatomic(crtc, ctx); if (crtc_state->hw.active || HAS_GMCH(dev_priv)) { @@ -18803,8 +19191,9 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) intel_crtc_free_hw_state(crtc_state); intel_crtc_state_reset(crtc_state, crtc); - crtc_state->hw.active = crtc_state->hw.enable = - dev_priv->display.get_pipe_config(crtc, crtc_state); + intel_crtc_get_pipe_config(crtc_state); + + crtc_state->hw.enable = crtc_state->hw.active; crtc->base.enabled = crtc_state->hw.enable; crtc->active = crtc_state->hw.active; @@ -18835,9 +19224,19 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) crtc_state = to_intel_crtc_state(crtc->base.state); encoder->base.crtc = &crtc->base; - encoder->get_config(encoder, crtc_state); + intel_encoder_get_config(encoder, crtc_state); if (encoder->sync_state) encoder->sync_state(encoder, crtc_state); + + /* read out to slave crtc as well for bigjoiner */ + if (crtc_state->bigjoiner) { + /* encoder should read be linked to bigjoiner master */ + WARN_ON(crtc_state->bigjoiner_slave); + + crtc = crtc_state->bigjoiner_linked_crtc; + crtc_state = to_intel_crtc_state(crtc->base.state); + intel_encoder_get_config(encoder, crtc_state); + } } else { encoder->base.crtc = NULL; } @@ -18893,16 +19292,10 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) struct intel_plane *plane; int min_cdclk = 0; - if (crtc_state->hw.active) { - struct drm_display_mode *mode = &crtc_state->hw.mode; - - intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode, - crtc_state); - - *mode = crtc_state->hw.adjusted_mode; - mode->hdisplay = crtc_state->pipe_src_w; - mode->vdisplay = crtc_state->pipe_src_h; + if (crtc_state->bigjoiner_slave) + continue; + if (crtc_state->hw.active) { /* * The initial mode needs to be set in order to keep * the atomic core happy. It wants a valid mode if the @@ -18914,8 +19307,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) */ crtc_state->inherited = true; - intel_crtc_compute_pixel_rate(crtc_state); - intel_crtc_update_active_timings(crtc_state); intel_crtc_copy_hw_to_uapi_state(crtc_state); @@ -18964,6 +19355,39 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) intel_bw_crtc_update(bw_state, crtc_state); intel_pipe_config_sanity_check(dev_priv, crtc_state); + + /* discard our incomplete slave state, copy it from master */ + if (crtc_state->bigjoiner && crtc_state->hw.active) { + struct intel_crtc *slave = crtc_state->bigjoiner_linked_crtc; + struct intel_crtc_state *slave_crtc_state = + to_intel_crtc_state(slave->base.state); + + copy_bigjoiner_crtc_state(slave_crtc_state, crtc_state); + slave->base.mode = crtc->base.mode; + + cdclk_state->min_cdclk[slave->pipe] = min_cdclk; + cdclk_state->min_voltage_level[slave->pipe] = + crtc_state->min_voltage_level; + + for_each_intel_plane_on_crtc(&dev_priv->drm, slave, plane) { + const struct intel_plane_state *plane_state = + to_intel_plane_state(plane->base.state); + + /* + * FIXME don't have the fb yet, so can't + * use intel_plane_data_rate() :( + */ + if (plane_state->uapi.visible) + crtc_state->data_rate[plane->id] = + 4 * crtc_state->pixel_rate; + else + crtc_state->data_rate[plane->id] = 0; + } + + intel_bw_crtc_update(bw_state, slave_crtc_state); + drm_calc_timestamping_constants(&slave->base, + &slave_crtc_state->hw.adjusted_mode); + } } } diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index be774f216065..5e0d42d82c11 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -513,7 +513,8 @@ u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv, bool intel_plane_can_remap(const struct intel_plane_state *plane_state); enum drm_mode_status intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv, - const struct drm_display_mode *mode); + const struct drm_display_mode *mode, + bool bigjoiner); enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port); bool is_trans_port_sync_mode(const struct intel_crtc_state *state); @@ -590,8 +591,8 @@ void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe); int lpt_get_iclkip(struct drm_i915_private *dev_priv); bool intel_fuzzy_clock_check(int clock1, int clock2); -void intel_prepare_reset(struct drm_i915_private *dev_priv); -void intel_finish_reset(struct drm_i915_private *dev_priv); +void intel_display_prepare_reset(struct drm_i915_private *dev_priv); +void intel_display_finish_reset(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config); void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, @@ -609,8 +610,6 @@ enum intel_display_power_domain intel_aux_power_domain(struct intel_digital_port *dig_port); enum intel_display_power_domain intel_legacy_aux_to_power_domain(enum aux_ch aux_ch); -void intel_mode_from_pipe_config(struct drm_display_mode *mode, - struct intel_crtc_state *pipe_config); void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index cfb4c1474982..ca41e8c00ad7 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -755,6 +755,17 @@ static void plane_rotation(char *buf, size_t bufsize, unsigned int rotation) rotation); } +static const char *plane_visibility(const struct intel_plane_state *plane_state) +{ + if (plane_state->uapi.visible) + return "visible"; + + if (plane_state->planar_slave) + return "planar-slave"; + + return "hidden"; +} + static void intel_plane_uapi_info(struct seq_file *m, struct intel_plane *plane) { const struct intel_plane_state *plane_state = @@ -773,12 +784,19 @@ static void intel_plane_uapi_info(struct seq_file *m, struct intel_plane *plane) plane_rotation(rot_str, sizeof(rot_str), plane_state->uapi.rotation); - seq_printf(m, "\t\tuapi: fb=%d,%s,%dx%d, src=" DRM_RECT_FP_FMT ", dst=" DRM_RECT_FMT ", rotation=%s\n", + seq_printf(m, "\t\tuapi: [FB:%d] %s,0x%llx,%dx%d, visible=%s, src=" DRM_RECT_FP_FMT ", dst=" DRM_RECT_FMT ", rotation=%s\n", fb ? fb->base.id : 0, fb ? format_name.str : "n/a", + fb ? fb->modifier : 0, fb ? fb->width : 0, fb ? fb->height : 0, + plane_visibility(plane_state), DRM_RECT_FP_ARG(&src), DRM_RECT_ARG(&dst), rot_str); + + if (plane_state->planar_linked_plane) + seq_printf(m, "\t\tplanar: Linked to [PLANE:%d:%s] as a %s\n", + plane_state->planar_linked_plane->base.base.id, plane_state->planar_linked_plane->base.name, + plane_state->planar_slave ? "slave" : "master"); } static void intel_plane_hw_info(struct seq_file *m, struct intel_plane *plane) @@ -797,9 +815,9 @@ static void intel_plane_hw_info(struct seq_file *m, struct intel_plane *plane) plane_rotation(rot_str, sizeof(rot_str), plane_state->hw.rotation); - seq_printf(m, "\t\thw: fb=%d,%s,%dx%d, visible=%s, src=" DRM_RECT_FP_FMT ", dst=" DRM_RECT_FMT ", rotation=%s\n", + seq_printf(m, "\t\thw: [FB:%d] %s,0x%llx,%dx%d, visible=%s, src=" DRM_RECT_FP_FMT ", dst=" DRM_RECT_FMT ", rotation=%s\n", fb->base.id, format_name.str, - fb->width, fb->height, + fb->modifier, fb->width, fb->height, yesno(plane_state->uapi.visible), DRM_RECT_FP_ARG(&plane_state->uapi.src), DRM_RECT_ARG(&plane_state->uapi.dst), @@ -874,6 +892,12 @@ static void intel_crtc_info(struct seq_file *m, struct intel_crtc *crtc) intel_scaler_info(m, crtc); } + if (crtc_state->bigjoiner) + seq_printf(m, "\tLinked to [CRTC:%d:%s] as a %s\n", + crtc_state->bigjoiner_linked_crtc->base.base.id, + crtc_state->bigjoiner_linked_crtc->base.name, + crtc_state->bigjoiner_slave ? "slave" : "master"); + for_each_intel_encoder_mask(&dev_priv->drm, encoder, crtc_state->uapi.encoder_mask) intel_encoder_info(m, crtc, encoder); diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 689922480661..fe2d90bba536 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -4492,30 +4492,24 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv, int requested_dc; int max_dc; - if (INTEL_GEN(dev_priv) >= 12) { - if (IS_DG1(dev_priv)) - max_dc = 3; - else - max_dc = 4; - /* - * DC9 has a separate HW flow from the rest of the DC states, - * not depending on the DMC firmware. It's needed by system - * suspend/resume, so allow it unconditionally. - */ - mask = DC_STATE_EN_DC9; - } else if (IS_GEN(dev_priv, 11)) { + if (IS_DG1(dev_priv)) + max_dc = 3; + else if (INTEL_GEN(dev_priv) >= 12) + max_dc = 4; + else if (INTEL_GEN(dev_priv) >= 10 || IS_GEN9_BC(dev_priv)) max_dc = 2; - mask = DC_STATE_EN_DC9; - } else if (IS_GEN(dev_priv, 10) || IS_GEN9_BC(dev_priv)) { - max_dc = 2; - mask = 0; - } else if (IS_GEN9_LP(dev_priv)) { + else if (IS_GEN9_LP(dev_priv)) max_dc = 1; - mask = DC_STATE_EN_DC9; - } else { + else max_dc = 0; - mask = 0; - } + + /* + * DC9 has a separate HW flow from the rest of the DC states, + * not depending on the DMC firmware. It's needed by system + * suspend/resume, so allow it unconditionally. + */ + mask = IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 11 ? + DC_STATE_EN_DC9 : 0; if (!dev_priv->params.disable_power_well) max_dc = 0; @@ -5858,10 +5852,15 @@ static void intel_power_domains_verify_state(struct drm_i915_private *i915) void intel_display_power_suspend_late(struct drm_i915_private *i915) { - if (INTEL_GEN(i915) >= 11 || IS_GEN9_LP(i915)) + if (INTEL_GEN(i915) >= 11 || IS_GEN9_LP(i915)) { bxt_enable_dc9(i915); - else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + /* Tweaked Wa_14010685332:icp,jsp,mcc */ + if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC) + intel_de_rmw(i915, SOUTH_CHICKEN1, + SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS); + } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_enable_pc8(i915); + } } void intel_display_power_resume_early(struct drm_i915_private *i915) @@ -5869,6 +5868,10 @@ void intel_display_power_resume_early(struct drm_i915_private *i915) if (INTEL_GEN(i915) >= 11 || IS_GEN9_LP(i915)) { gen9_sanitize_dc_state(i915); bxt_disable_dc9(i915); + /* Tweaked Wa_14010685332:icp,jsp,mcc */ + if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC) + intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0); + } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_disable_pc8(i915); } diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index f6f0626649e0..ce82d654d0f2 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -686,6 +686,7 @@ struct skl_wm_level { u8 plane_res_l; bool plane_en; bool ignore_lines; + bool can_sagv; }; struct skl_plane_wm { @@ -737,24 +738,35 @@ struct g4x_wm_state { struct intel_crtc_wm_state { union { + /* + * raw: + * The "raw" watermark values produced by the formula + * given the plane's current state. They do not consider + * how much FIFO is actually allocated for each plane. + * + * optimal: + * The "optimal" watermark values given the current + * state of the planes and the amount of FIFO + * allocated to each, ignoring any previous state + * of the planes. + * + * intermediate: + * The "intermediate" watermark values when transitioning + * between the old and new "optimal" values. Used when + * the watermark registers are single buffered and hence + * their state changes asynchronously with regards to the + * actual plane registers. These are essentially the + * worst case combination of the old and new "optimal" + * watermarks, which are therefore safe to use when the + * plane is in either its old or new state. + */ struct { - /* - * Intermediate watermarks; these can be - * programmed immediately since they satisfy - * both the current configuration we're - * switching away from and the new - * configuration we're switching to. - */ struct intel_pipe_wm intermediate; - - /* - * Optimal watermarks, programmed post-vblank - * when this state is committed. - */ struct intel_pipe_wm optimal; } ilk; struct { + struct skl_pipe_wm raw; /* gen9+ only needs 1-step wm programming */ struct skl_pipe_wm optimal; struct skl_ddb_entry ddb; @@ -763,22 +775,15 @@ struct intel_crtc_wm_state { } skl; struct { - /* "raw" watermarks (not inverted) */ - struct g4x_pipe_wm raw[NUM_VLV_WM_LEVELS]; - /* intermediate watermarks (inverted) */ - struct vlv_wm_state intermediate; - /* optimal watermarks (inverted) */ - struct vlv_wm_state optimal; - /* display FIFO split */ + struct g4x_pipe_wm raw[NUM_VLV_WM_LEVELS]; /* not inverted */ + struct vlv_wm_state intermediate; /* inverted */ + struct vlv_wm_state optimal; /* inverted */ struct vlv_fifo_state fifo_state; } vlv; struct { - /* "raw" watermarks */ struct g4x_pipe_wm raw[NUM_G4X_WM_LEVELS]; - /* intermediate watermarks */ struct g4x_wm_state intermediate; - /* optimal watermarks */ struct g4x_wm_state optimal; } g4x; }; @@ -817,15 +822,22 @@ struct intel_crtc_state { * The following members are used to verify the hardware state: * - enable * - active - * - mode / adjusted_mode + * - mode / pipe_mode / adjusted_mode * - color property blobs. * * During initial hw readout, they need to be copied to uapi. + * + * Bigjoiner will allow a transcoder mode that spans 2 pipes; + * Use the pipe_mode for calculations like watermarks, pipe + * scaler, and bandwidth. + * + * Use adjusted_mode for things that need to know the full + * mode on the transcoder, which spans all pipes. */ struct { bool active, enable; struct drm_property_blob *degamma_lut, *gamma_lut, *ctm; - struct drm_display_mode mode, adjusted_mode; + struct drm_display_mode mode, pipe_mode, adjusted_mode; enum drm_scaling_filter scaling_filter; } hw; @@ -838,6 +850,7 @@ struct intel_crtc_state { * accordingly. */ #define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */ +#define PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE (1<<1) /* bigjoiner slave, partial readout */ unsigned long quirks; unsigned fb_bits; /* framebuffers to flip */ @@ -1019,6 +1032,10 @@ struct intel_crtc_state { u32 data_rate[I915_MAX_PLANES]; + /* FIXME unify with data_rate[] */ + u64 plane_data_rate[I915_MAX_PLANES]; + u64 uv_plane_data_rate[I915_MAX_PLANES]; + /* Gamma mode programmed on the pipe */ u32 gamma_mode; @@ -1063,6 +1080,15 @@ struct intel_crtc_state { /* enable pipe csc? */ bool csc_enable; + /* enable pipe big joiner? */ + bool bigjoiner; + + /* big joiner slave crtc? */ + bool bigjoiner_slave; + + /* linked crtc for bigjoiner, either slave or master */ + struct intel_crtc *bigjoiner_linked_crtc; + /* Display Stream compression state */ struct { bool compression_enable; @@ -1189,6 +1215,15 @@ struct intel_plane { * the intel_plane_state structure and accessed via plane_state. */ + int (*min_width)(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation); + int (*max_width)(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation); + int (*max_height)(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation); unsigned int (*max_stride)(struct intel_plane *plane, u32 pixel_format, u64 modifier, unsigned int rotation); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index cf09aca7607b..2165398d2c7c 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -254,6 +254,17 @@ intel_dp_max_data_rate(int max_link_clock, int max_lanes) return max_link_clock * max_lanes; } +bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &intel_dig_port->base; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + + return INTEL_GEN(dev_priv) >= 12 || + (INTEL_GEN(dev_priv) == 11 && + encoder->port != PORT_A); +} + static int cnl_max_source_rate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); @@ -519,7 +530,8 @@ small_joiner_ram_size_bits(struct drm_i915_private *i915) static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, u32 link_clock, u32 lane_count, - u32 mode_clock, u32 mode_hdisplay) + u32 mode_clock, u32 mode_hdisplay, + bool bigjoiner) { u32 bits_per_pixel, max_bpp_small_joiner_ram; int i; @@ -537,6 +549,10 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, /* Small Joiner Check: output bpp <= joiner RAM (bits) / Horiz. width */ max_bpp_small_joiner_ram = small_joiner_ram_size_bits(i915) / mode_hdisplay; + + if (bigjoiner) + max_bpp_small_joiner_ram *= 2; + drm_dbg_kms(&i915->drm, "Max small joiner bpp: %u\n", max_bpp_small_joiner_ram); @@ -546,6 +562,15 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, */ bits_per_pixel = min(bits_per_pixel, max_bpp_small_joiner_ram); + if (bigjoiner) { + u32 max_bpp_bigjoiner = + i915->max_cdclk_freq * 48 / + intel_dp_mode_to_fec_clock(mode_clock); + + DRM_DEBUG_KMS("Max big joiner bpp: %u\n", max_bpp_bigjoiner); + bits_per_pixel = min(bits_per_pixel, max_bpp_bigjoiner); + } + /* Error out if the max bpp is less than smallest allowed valid bpp */ if (bits_per_pixel < valid_dsc_bpp[0]) { drm_dbg_kms(&i915->drm, "Unsupported BPP %u, min %u\n", @@ -568,7 +593,8 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, } static u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, - int mode_clock, int mode_hdisplay) + int mode_clock, int mode_hdisplay, + bool bigjoiner) { struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 min_slice_count, i; @@ -589,18 +615,24 @@ static u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, return 0; } /* Also take into account max slice width */ - min_slice_count = min_t(u8, min_slice_count, + min_slice_count = max_t(u8, min_slice_count, DIV_ROUND_UP(mode_hdisplay, max_slice_width)); /* Find the closest match to the valid slice count values */ for (i = 0; i < ARRAY_SIZE(valid_dsc_slicecount); i++) { - if (valid_dsc_slicecount[i] > - drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, - false)) + u8 test_slice_count = valid_dsc_slicecount[i] << bigjoiner; + + if (test_slice_count > + drm_dp_dsc_sink_max_slice_count(intel_dp->dsc_dpcd, false)) break; - if (min_slice_count <= valid_dsc_slicecount[i]) - return valid_dsc_slicecount[i]; + + /* big joiner needs small joiner to be enabled */ + if (bigjoiner && test_slice_count < 4) + continue; + + if (min_slice_count <= test_slice_count) + return test_slice_count; } drm_dbg_kms(&i915->drm, "Unsupported Slice Count %d\n", @@ -717,10 +749,14 @@ intel_dp_mode_valid(struct drm_connector *connector, u16 dsc_max_output_bpp = 0; u8 dsc_slice_count = 0; enum drm_mode_status status; + bool dsc = false, bigjoiner = false; if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; + if (mode->flags & DRM_MODE_FLAG_DBLCLK) + return MODE_H_ILLEGAL; + if (intel_dp_is_edp(intel_dp) && fixed_mode) { if (mode->hdisplay > fixed_mode->hdisplay) return MODE_PANEL; @@ -731,6 +767,17 @@ intel_dp_mode_valid(struct drm_connector *connector, target_clock = fixed_mode->clock; } + if (mode->clock < 10000) + return MODE_CLOCK_LOW; + + if ((target_clock > max_dotclk || mode->hdisplay > 5120) && + intel_dp_can_bigjoiner(intel_dp)) { + bigjoiner = true; + max_dotclk *= 2; + } + if (target_clock > max_dotclk) + return MODE_CLOCK_HIGH; + max_link_clock = intel_dp_max_link_rate(intel_dp); max_lanes = intel_dp_max_lane_count(intel_dp); @@ -759,30 +806,31 @@ intel_dp_mode_valid(struct drm_connector *connector, max_link_clock, max_lanes, target_clock, - mode->hdisplay) >> 4; + mode->hdisplay, + bigjoiner) >> 4; dsc_slice_count = intel_dp_dsc_get_slice_count(intel_dp, target_clock, - mode->hdisplay); + mode->hdisplay, + bigjoiner); } + + dsc = dsc_max_output_bpp && dsc_slice_count; } - if ((mode_rate > max_rate && !(dsc_max_output_bpp && dsc_slice_count)) || - target_clock > max_dotclk) + /* big joiner configuration needs DSC */ + if (bigjoiner && !dsc) return MODE_CLOCK_HIGH; - if (mode->clock < 10000) - return MODE_CLOCK_LOW; - - if (mode->flags & DRM_MODE_FLAG_DBLCLK) - return MODE_H_ILLEGAL; + if (mode_rate > max_rate && !dsc) + return MODE_CLOCK_HIGH; status = intel_dp_mode_valid_downstream(intel_connector, mode, target_clock); if (status != MODE_OK) return status; - return intel_mode_valid_max_plane_size(dev_priv, mode); + return intel_mode_valid_max_plane_size(dev_priv, mode, bigjoiner); } u32 intel_dp_pack_aux(const u8 *src, int src_bytes) @@ -2052,12 +2100,10 @@ static bool intel_dp_supports_fec(struct intel_dp *intel_dp, static bool intel_dp_supports_dsc(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { - struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - - if (!intel_dp_is_edp(intel_dp) && !crtc_state->fec_enable) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP) && !crtc_state->fec_enable) return false; - return intel_dsc_source_support(encoder, crtc_state) && + return intel_dsc_source_support(crtc_state) && drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd); } @@ -2351,11 +2397,13 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, pipe_config->port_clock, pipe_config->lane_count, adjusted_mode->crtc_clock, - adjusted_mode->crtc_hdisplay); + adjusted_mode->crtc_hdisplay, + pipe_config->bigjoiner); dsc_dp_slice_count = intel_dp_dsc_get_slice_count(intel_dp, adjusted_mode->crtc_clock, - adjusted_mode->crtc_hdisplay); + adjusted_mode->crtc_hdisplay, + pipe_config->bigjoiner); if (!dsc_max_output_bpp || !dsc_dp_slice_count) { drm_dbg_kms(&dev_priv->drm, "Compressed BPP/Slice Count not supported\n"); @@ -2371,14 +2419,15 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, * is greater than the maximum Cdclock and if slice count is even * then we need to use 2 VDSC instances. */ - if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq) { - if (pipe_config->dsc.slice_count > 1) { - pipe_config->dsc.dsc_split = true; - } else { + if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq || + pipe_config->bigjoiner) { + if (pipe_config->dsc.slice_count < 2) { drm_dbg_kms(&dev_priv->drm, "Cannot split stream to use 2 VDSC instances\n"); return -EINVAL; } + + pipe_config->dsc.dsc_split = true; } ret = intel_dp_dsc_compute_params(&dig_port->base, pipe_config); @@ -2449,6 +2498,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, intel_dp->common_rates[limits.max_clock], limits.max_bpp, adjusted_mode->crtc_clock); + if ((adjusted_mode->crtc_clock > i915->max_dotclk_freq || + adjusted_mode->crtc_hdisplay > 5120) && + intel_dp_can_bigjoiner(intel_dp)) + pipe_config->bigjoiner = true; + /* * Optimize for slow and wide. This is the place to add alternative * optimization policy. @@ -2457,7 +2511,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, /* enable compression if the mode doesn't fit available BW */ drm_dbg_kms(&i915->drm, "Force DSC en = %d\n", intel_dp->force_dsc_en); - if (ret || intel_dp->force_dsc_en) { + if (ret || intel_dp->force_dsc_en || pipe_config->bigjoiner) { ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, conn_state, &limits); if (ret < 0) @@ -3778,6 +3832,12 @@ bool intel_dp_initial_fastset_check(struct intel_encoder *encoder, return false; } + if (CAN_PSR(i915) && intel_dp_is_edp(intel_dp)) { + drm_dbg_kms(&i915->drm, "Forcing full modeset to compute PSR state\n"); + crtc_state->uapi.mode_changed = true; + return false; + } + return true; } diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h index 3f862b4fd34f..b871a09b6901 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.h +++ b/drivers/gpu/drm/i915/display/intel_dp.h @@ -106,6 +106,7 @@ bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp); bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp); int intel_dp_link_required(int pixel_clock, int bpp); int intel_dp_max_data_rate(int max_link_clock, int max_lanes); +bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp); bool intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp, diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 4f05ffa3e761..27f04aed8764 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -715,7 +715,7 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector, return 0; } - *status = intel_mode_valid_max_plane_size(dev_priv, mode); + *status = intel_mode_valid_max_plane_size(dev_priv, mode, false); return 0; } diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index a1b4e7110952..f6ad257a260e 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -141,7 +141,7 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, "asserting DPLL %s with no DPLL\n", onoff(state))) return; - cur_state = pll->info->funcs->get_hw_state(dev_priv, pll, &hw_state); + cur_state = intel_dpll_get_hw_state(dev_priv, pll, &hw_state); I915_STATE_WARN(cur_state != state, "%s assertion failure (expected %s, current %s)\n", pll->info->name, onoff(state), onoff(cur_state)); @@ -891,11 +891,12 @@ hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state, } static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { int refclk; int n, p, r; - u32 wrpll = pll->state.hw_state.wrpll; + u32 wrpll = pll_state->wrpll; switch (wrpll & WRPLL_REF_MASK) { case WRPLL_REF_SPECIAL_HSW: @@ -962,7 +963,8 @@ hsw_ddi_lcpll_get_dpll(struct intel_crtc_state *crtc_state) } static int hsw_ddi_lcpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { int link_clock = 0; @@ -1002,11 +1004,12 @@ hsw_ddi_spll_get_dpll(struct intel_atomic_state *state, } static int hsw_ddi_spll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { int link_clock = 0; - switch (pll->state.hw_state.spll & SPLL_FREQ_MASK) { + switch (pll_state->spll & SPLL_FREQ_MASK) { case SPLL_FREQ_810MHz: link_clock = 81000; break; @@ -1577,9 +1580,9 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) } static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; int ref_clock = i915->dpll.ref_clks.nssc; u32 p0, p1, p2, dco_freq; @@ -1688,12 +1691,12 @@ skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) } static int skl_ddi_lcpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { int link_clock = 0; - switch ((pll->state.hw_state.ctrl1 & - DPLL_CTRL1_LINK_RATE_MASK(0)) >> + switch ((pll_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0)) >> DPLL_CTRL1_LINK_RATE_SHIFT(0)) { case DPLL_CTRL1_LINK_RATE_810: link_clock = 81000; @@ -1771,16 +1774,17 @@ static bool skl_get_dpll(struct intel_atomic_state *state, } static int skl_ddi_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { /* * ctrl1 register is already shifted for each pll, just use 0 to get * the internal shift for each field */ - if (pll->state.hw_state.ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) - return skl_ddi_wrpll_get_freq(i915, pll); + if (pll_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) + return skl_ddi_wrpll_get_freq(i915, pll, pll_state); else - return skl_ddi_lcpll_get_freq(i915, pll); + return skl_ddi_lcpll_get_freq(i915, pll, pll_state); } static void skl_update_dpll_ref_clks(struct drm_i915_private *i915) @@ -2218,9 +2222,9 @@ bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state) } static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; struct dpll clock; clock.m1 = 2; @@ -2636,20 +2640,23 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) } /* - * Display WA #22010492432: tgl + * Display WA #22010492432: ehl, tgl * Program half of the nominal DCO divider fraction value. */ static bool -tgl_combo_pll_div_frac_wa_needed(struct drm_i915_private *i915) +ehl_combo_pll_div_frac_wa_needed(struct drm_i915_private *i915) { - return IS_TIGERLAKE(i915) && i915->dpll.ref_clks.nssc == 38400; + return ((IS_PLATFORM(i915, INTEL_ELKHARTLAKE) && + IS_JSL_EHL_REVID(i915, EHL_REVID_B0, REVID_FOREVER)) || + IS_TIGERLAKE(i915)) && + i915->dpll.ref_clks.nssc == 38400; } static int __cnl_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state, int ref_clock) { - const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; u32 dco_fraction; u32 p0, p1, p2, dco_freq; @@ -2696,7 +2703,7 @@ static int __cnl_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, dco_fraction = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> DPLL_CFGCR0_DCO_FRACTION_SHIFT; - if (tgl_combo_pll_div_frac_wa_needed(dev_priv)) + if (ehl_combo_pll_div_frac_wa_needed(dev_priv)) dco_fraction *= 2; dco_freq += (dco_fraction * ref_clock) / 0x8000; @@ -2708,9 +2715,11 @@ static int __cnl_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, } static int cnl_ddi_wrpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - return __cnl_ddi_wrpll_get_freq(i915, pll, i915->dpll.ref_clks.nssc); + return __cnl_ddi_wrpll_get_freq(i915, pll, pll_state, + i915->dpll.ref_clks.nssc); } static bool @@ -2759,11 +2768,12 @@ cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) } static int cnl_ddi_lcpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { int link_clock = 0; - switch (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK) { + switch (pll_state->cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK) { case DPLL_CFGCR0_LINK_RATE_810: link_clock = 81000; break; @@ -2846,12 +2856,13 @@ static bool cnl_get_dpll(struct intel_atomic_state *state, } static int cnl_ddi_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - if (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_HDMI_MODE) - return cnl_ddi_wrpll_get_freq(i915, pll); + if (pll_state->cfgcr0 & DPLL_CFGCR0_HDMI_MODE) + return cnl_ddi_wrpll_get_freq(i915, pll, pll_state); else - return cnl_ddi_lcpll_get_freq(i915, pll); + return cnl_ddi_lcpll_get_freq(i915, pll, pll_state); } static void cnl_update_dpll_ref_clks(struct drm_i915_private *i915) @@ -3036,7 +3047,8 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, } static int icl_ddi_tbt_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { /* * The PLL outputs multiple frequencies at the same time, selection is @@ -3072,9 +3084,10 @@ icl_calc_wrpll(struct intel_crtc_state *crtc_state, } static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - return __cnl_ddi_wrpll_get_freq(i915, pll, + return __cnl_ddi_wrpll_get_freq(i915, pll, pll_state, icl_wrpll_ref_clock(i915)); } @@ -3086,7 +3099,7 @@ static void icl_calc_dpll_state(struct drm_i915_private *i915, memset(pll_state, 0, sizeof(*pll_state)); - if (tgl_combo_pll_div_frac_wa_needed(i915)) + if (ehl_combo_pll_div_frac_wa_needed(i915)) dco_fraction = DIV_ROUND_CLOSEST(dco_fraction, 2); pll_state->cfgcr0 = DPLL_CFGCR0_DCO_FRACTION(dco_fraction) | @@ -3399,9 +3412,9 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, } static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *dev_priv, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { - const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; u32 m1, m2_int, m2_frac, div1, div2, ref_clock; u64 tmp; @@ -4512,16 +4525,33 @@ void intel_update_active_dpll(struct intel_atomic_state *state, * intel_dpll_get_freq - calculate the DPLL's output frequency * @i915: i915 device * @pll: DPLL for which to calculate the output frequency + * @pll_state: DPLL state from which to calculate the output frequency * - * Return the output frequency corresponding to @pll's current state. + * Return the output frequency corresponding to @pll's passed in @pll_state. */ int intel_dpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll) + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) { if (drm_WARN_ON(&i915->drm, !pll->info->funcs->get_freq)) return 0; - return pll->info->funcs->get_freq(i915, pll); + return pll->info->funcs->get_freq(i915, pll, pll_state); +} + +/** + * intel_dpll_get_hw_state - readout the DPLL's hardware state + * @i915: i915 device + * @pll: DPLL for which to calculate the output frequency + * @hw_state: DPLL's hardware state + * + * Read out @pll's hardware state into @hw_state. + */ +bool intel_dpll_get_hw_state(struct drm_i915_private *i915, + struct intel_shared_dpll *pll, + struct intel_dpll_hw_state *hw_state) +{ + return pll->info->funcs->get_hw_state(i915, pll, hw_state); } static void readout_dpll_hw_state(struct drm_i915_private *i915, @@ -4529,8 +4559,7 @@ static void readout_dpll_hw_state(struct drm_i915_private *i915, { struct intel_crtc *crtc; - pll->on = pll->info->funcs->get_hw_state(i915, pll, - &pll->state.hw_state); + pll->on = intel_dpll_get_hw_state(i915, pll, &pll->state.hw_state); if (IS_JSL_EHL(i915) && pll->on && pll->info->id == DPLL_ID_EHL_DPLL4) { diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h index 205542fb8dc7..2eb7618ef957 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h @@ -300,10 +300,11 @@ struct intel_shared_dpll_funcs { * @get_freq: * * Hook for calculating the pll's output frequency based on its - * current state. + * passed in state. */ int (*get_freq)(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll); + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state); }; /** @@ -399,7 +400,11 @@ void intel_update_active_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder); int intel_dpll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll); + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state); +bool intel_dpll_get_hw_state(struct drm_i915_private *i915, + struct intel_shared_dpll *pll, + struct intel_dpll_hw_state *hw_state); void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c index afa4e6817e8c..f453ceb8d149 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.c +++ b/drivers/gpu/drm/i915/display/intel_dsi.c @@ -75,7 +75,7 @@ enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, return MODE_CLOCK_HIGH; } - return intel_mode_valid_max_plane_size(dev_priv, mode); + return intel_mode_valid_max_plane_size(dev_priv, mode, false); } struct intel_dsi_host *intel_dsi_host_init(struct intel_dsi *intel_dsi, diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index f90838bc74fb..82674a8853c6 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -2274,7 +2274,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector, if (status != MODE_OK) return status; - return intel_mode_valid_max_plane_size(dev_priv, mode); + return intel_mode_valid_max_plane_size(dev_priv, mode, false); } bool intel_hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 1576c3722d0b..b3631b722de3 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -1024,8 +1024,6 @@ void intel_psr_enable(struct intel_dp *intel_dp, if (!CAN_PSR(dev_priv) || dev_priv->psr.dp != intel_dp) return; - dev_priv->psr.force_mode_changed = false; - if (!crtc_state->has_psr) return; @@ -1334,8 +1332,6 @@ void intel_psr_update(struct intel_dp *intel_dp, if (!CAN_PSR(dev_priv) || READ_ONCE(psr->dp) != intel_dp) return; - dev_priv->psr.force_mode_changed = false; - mutex_lock(&dev_priv->psr.lock); enable = crtc_state->has_psr; @@ -1869,40 +1865,3 @@ bool intel_psr_enabled(struct intel_dp *intel_dp) return ret; } - -void intel_psr_atomic_check(struct drm_connector *connector, - struct drm_connector_state *old_state, - struct drm_connector_state *new_state) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - struct intel_connector *intel_connector; - struct intel_digital_port *dig_port; - struct drm_crtc_state *crtc_state; - - if (!CAN_PSR(dev_priv) || !new_state->crtc || - !dev_priv->psr.force_mode_changed) - return; - - intel_connector = to_intel_connector(connector); - dig_port = enc_to_dig_port(to_intel_encoder(new_state->best_encoder)); - if (dev_priv->psr.dp != &dig_port->dp) - return; - - crtc_state = drm_atomic_get_new_crtc_state(new_state->state, - new_state->crtc); - crtc_state->mode_changed = true; -} - -void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp) -{ - struct drm_i915_private *dev_priv; - - if (!intel_dp) - return; - - dev_priv = dp_to_i915(intel_dp); - if (!CAN_PSR(dev_priv) || intel_dp != dev_priv->psr.dp) - return; - - dev_priv->psr.force_mode_changed = true; -} diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h index 3eca9dcec3c0..0a517978e8af 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.h +++ b/drivers/gpu/drm/i915/display/intel_psr.h @@ -43,10 +43,6 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp); int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state, u32 *out_value); bool intel_psr_enabled(struct intel_dp *intel_dp); -void intel_psr_atomic_check(struct drm_connector *connector, - struct drm_connector_state *old_state, - struct drm_connector_state *new_state); -void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp); int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc *crtc); void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index b6deeb338477..019a2d6d807a 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -408,6 +408,134 @@ static int skl_plane_min_cdclk(const struct intel_crtc_state *crtc_state, return DIV_ROUND_UP(pixel_rate * num, den); } +static int skl_plane_max_width(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + int cpp = fb->format->cpp[color_plane]; + + switch (fb->modifier) { + case DRM_FORMAT_MOD_LINEAR: + case I915_FORMAT_MOD_X_TILED: + /* + * Validated limit is 4k, but has 5k should + * work apart from the following features: + * - Ytile (already limited to 4k) + * - FP16 (already limited to 4k) + * - render compression (already limited to 4k) + * - KVMR sprite and cursor (don't care) + * - horizontal panning (TODO verify this) + * - pipe and plane scaling (TODO verify this) + */ + if (cpp == 8) + return 4096; + else + return 5120; + case I915_FORMAT_MOD_Y_TILED_CCS: + case I915_FORMAT_MOD_Yf_TILED_CCS: + case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: + /* FIXME AUX plane? */ + case I915_FORMAT_MOD_Y_TILED: + case I915_FORMAT_MOD_Yf_TILED: + if (cpp == 8) + return 2048; + else + return 4096; + default: + MISSING_CASE(fb->modifier); + return 2048; + } +} + +static int glk_plane_max_width(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + int cpp = fb->format->cpp[color_plane]; + + switch (fb->modifier) { + case DRM_FORMAT_MOD_LINEAR: + case I915_FORMAT_MOD_X_TILED: + if (cpp == 8) + return 4096; + else + return 5120; + case I915_FORMAT_MOD_Y_TILED_CCS: + case I915_FORMAT_MOD_Yf_TILED_CCS: + /* FIXME AUX plane? */ + case I915_FORMAT_MOD_Y_TILED: + case I915_FORMAT_MOD_Yf_TILED: + if (cpp == 8) + return 2048; + else + return 5120; + default: + MISSING_CASE(fb->modifier); + return 2048; + } +} + +static int icl_plane_min_width(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + /* Wa_14011264657, Wa_14011050563: gen11+ */ + switch (fb->format->format) { + case DRM_FORMAT_C8: + return 18; + case DRM_FORMAT_RGB565: + return 10; + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ARGB8888: + case DRM_FORMAT_ABGR8888: + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_XBGR2101010: + case DRM_FORMAT_ARGB2101010: + case DRM_FORMAT_ABGR2101010: + case DRM_FORMAT_XVYU2101010: + case DRM_FORMAT_Y212: + case DRM_FORMAT_Y216: + return 6; + case DRM_FORMAT_NV12: + return 20; + case DRM_FORMAT_P010: + case DRM_FORMAT_P012: + case DRM_FORMAT_P016: + return 12; + case DRM_FORMAT_XRGB16161616F: + case DRM_FORMAT_XBGR16161616F: + case DRM_FORMAT_ARGB16161616F: + case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XVYU12_16161616: + case DRM_FORMAT_XVYU16161616: + return 4; + default: + return 1; + } +} + +static int icl_plane_max_width(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + return 5120; +} + +static int skl_plane_max_height(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + return 4096; +} + +static int icl_plane_max_height(const struct drm_framebuffer *fb, + int color_plane, + unsigned int rotation) +{ + return 4320; +} + static unsigned int skl_plane_max_stride(struct intel_plane *plane, u32 pixel_format, u64 modifier, @@ -2059,10 +2187,8 @@ g4x_sprite_check(struct intel_crtc_state *crtc_state, } } - ret = drm_atomic_helper_check_plane_state(&plane_state->uapi, - &crtc_state->uapi, - min_scale, max_scale, - true, true); + ret = intel_atomic_plane_check_clipping(plane_state, crtc_state, + min_scale, max_scale, true); if (ret) return ret; @@ -2117,11 +2243,10 @@ vlv_sprite_check(struct intel_crtc_state *crtc_state, if (ret) return ret; - ret = drm_atomic_helper_check_plane_state(&plane_state->uapi, - &crtc_state->uapi, - DRM_PLANE_HELPER_NO_SCALING, - DRM_PLANE_HELPER_NO_SCALING, - true, true); + ret = intel_atomic_plane_check_clipping(plane_state, crtc_state, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + true); if (ret) return ret; @@ -2328,10 +2453,8 @@ static int skl_plane_check(struct intel_crtc_state *crtc_state, max_scale = skl_plane_max_scale(dev_priv, fb); } - ret = drm_atomic_helper_check_plane_state(&plane_state->uapi, - &crtc_state->uapi, - min_scale, max_scale, - true, true); + ret = intel_atomic_plane_check_clipping(plane_state, crtc_state, + min_scale, max_scale, true); if (ret) return ret; @@ -3133,6 +3256,18 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, fbc->possible_framebuffer_bits |= plane->frontbuffer_bit; } + if (INTEL_GEN(dev_priv) >= 11) { + plane->min_width = icl_plane_min_width; + plane->max_width = icl_plane_max_width; + plane->max_height = icl_plane_max_height; + } else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) { + plane->max_width = glk_plane_max_width; + plane->max_height = skl_plane_max_height; + } else { + plane->max_width = skl_plane_max_width; + plane->max_height = skl_plane_max_height; + } + plane->max_stride = skl_plane_max_stride; plane->update_plane = skl_update_plane; plane->disable_plane = skl_disable_plane; diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index c5735c365659..e2716a67b281 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -332,11 +332,10 @@ static const struct rc_parameters *get_rc_params(u16 compressed_bpp, return &rc_parameters[row_index][column_index]; } -bool intel_dsc_source_support(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) +bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state) { const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *i915 = to_i915(encoder->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum pipe pipe = crtc->pipe; @@ -490,11 +489,10 @@ intel_dsc_power_domain(const struct intel_crtc_state *crtc_state) return POWER_DOMAIN_TRANSCODER_VDSC_PW2; } -static void intel_dsc_pps_configure(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state) +static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; enum pipe pipe = crtc->pipe; u32 pps_val = 0; @@ -503,6 +501,9 @@ static void intel_dsc_pps_configure(struct intel_encoder *encoder, u8 num_vdsc_instances = (crtc_state->dsc.dsc_split) ? 2 : 1; int i = 0; + if (crtc_state->bigjoiner) + num_vdsc_instances *= 2; + /* Populate PICTURE_PARAMETER_SET_0 registers */ pps_val = DSC_VER_MAJ | vdsc_cfg->dsc_version_minor << DSC_VER_MIN_SHIFT | @@ -973,55 +974,6 @@ static void intel_dsc_pps_configure(struct intel_encoder *encoder, } } -void intel_dsc_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - enum pipe pipe = crtc->pipe; - enum intel_display_power_domain power_domain; - intel_wakeref_t wakeref; - u32 dss_ctl1, dss_ctl2, val; - - if (!intel_dsc_source_support(encoder, crtc_state)) - return; - - power_domain = intel_dsc_power_domain(crtc_state); - - wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); - if (!wakeref) - return; - - if (!is_pipe_dsc(crtc_state)) { - dss_ctl1 = intel_de_read(dev_priv, DSS_CTL1); - dss_ctl2 = intel_de_read(dev_priv, DSS_CTL2); - } else { - dss_ctl1 = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL1(pipe)); - dss_ctl2 = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL2(pipe)); - } - - crtc_state->dsc.compression_enable = dss_ctl2 & LEFT_BRANCH_VDSC_ENABLE; - if (!crtc_state->dsc.compression_enable) - goto out; - - crtc_state->dsc.dsc_split = (dss_ctl2 & RIGHT_BRANCH_VDSC_ENABLE) && - (dss_ctl1 & JOINER_ENABLE); - - /* FIXME: add more state readout as needed */ - - /* PPS1 */ - if (!is_pipe_dsc(crtc_state)) - val = intel_de_read(dev_priv, DSCA_PICTURE_PARAMETER_SET_1); - else - val = intel_de_read(dev_priv, - ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe)); - vdsc_cfg->bits_per_pixel = val; - crtc_state->dsc.compressed_bpp = vdsc_cfg->bits_per_pixel >> 4; -out: - intel_display_power_put(dev_priv, power_domain, wakeref); -} - static void intel_dsc_dsi_pps_write(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { @@ -1060,77 +1012,126 @@ static void intel_dsc_dp_pps_write(struct intel_encoder *encoder, sizeof(dp_dsc_pps_sdp)); } +static i915_reg_t dss_ctl1_reg(const struct intel_crtc_state *crtc_state) +{ + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + + if (crtc_state->cpu_transcoder == TRANSCODER_EDP) + return DSS_CTL1; + + return ICL_PIPE_DSS_CTL1(pipe); +} + +static i915_reg_t dss_ctl2_reg(const struct intel_crtc_state *crtc_state) +{ + enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe; + + if (crtc_state->cpu_transcoder == TRANSCODER_EDP) + return DSS_CTL2; + + return ICL_PIPE_DSS_CTL2(pipe); +} + void intel_dsc_enable(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum pipe pipe = crtc->pipe; - i915_reg_t dss_ctl1_reg, dss_ctl2_reg; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dss_ctl1_val = 0; u32 dss_ctl2_val = 0; if (!crtc_state->dsc.compression_enable) return; - /* Enable Power wells for VDSC/joining */ - intel_display_power_get(dev_priv, - intel_dsc_power_domain(crtc_state)); - - intel_dsc_pps_configure(encoder, crtc_state); + intel_dsc_pps_configure(crtc_state); - if (encoder->type == INTEL_OUTPUT_DSI) - intel_dsc_dsi_pps_write(encoder, crtc_state); - else - intel_dsc_dp_pps_write(encoder, crtc_state); - - if (!is_pipe_dsc(crtc_state)) { - dss_ctl1_reg = DSS_CTL1; - dss_ctl2_reg = DSS_CTL2; - } else { - dss_ctl1_reg = ICL_PIPE_DSS_CTL1(pipe); - dss_ctl2_reg = ICL_PIPE_DSS_CTL2(pipe); + if (!crtc_state->bigjoiner_slave) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + intel_dsc_dsi_pps_write(encoder, crtc_state); + else + intel_dsc_dp_pps_write(encoder, crtc_state); } + dss_ctl2_val |= LEFT_BRANCH_VDSC_ENABLE; if (crtc_state->dsc.dsc_split) { dss_ctl2_val |= RIGHT_BRANCH_VDSC_ENABLE; dss_ctl1_val |= JOINER_ENABLE; } - intel_de_write(dev_priv, dss_ctl1_reg, dss_ctl1_val); - intel_de_write(dev_priv, dss_ctl2_reg, dss_ctl2_val); + if (crtc_state->bigjoiner) { + dss_ctl1_val |= BIG_JOINER_ENABLE; + if (!crtc_state->bigjoiner_slave) + dss_ctl1_val |= MASTER_BIG_JOINER_ENABLE; + } + intel_de_write(dev_priv, dss_ctl1_reg(crtc_state), dss_ctl1_val); + intel_de_write(dev_priv, dss_ctl2_reg(crtc_state), dss_ctl2_val); } void intel_dsc_disable(const struct intel_crtc_state *old_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - i915_reg_t dss_ctl1_reg, dss_ctl2_reg; - u32 dss_ctl1_val = 0, dss_ctl2_val = 0; if (!old_crtc_state->dsc.compression_enable) return; - if (!is_pipe_dsc(old_crtc_state)) { - dss_ctl1_reg = DSS_CTL1; - dss_ctl2_reg = DSS_CTL2; - } else { - dss_ctl1_reg = ICL_PIPE_DSS_CTL1(pipe); - dss_ctl2_reg = ICL_PIPE_DSS_CTL2(pipe); + intel_de_write(dev_priv, dss_ctl1_reg(old_crtc_state), 0); + intel_de_write(dev_priv, dss_ctl2_reg(old_crtc_state), 0); +} + +void intel_dsc_get_config(struct intel_crtc_state *crtc_state) +{ + struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + enum intel_display_power_domain power_domain; + intel_wakeref_t wakeref; + u32 dss_ctl1, dss_ctl2, val; + + if (!intel_dsc_source_support(crtc_state)) + return; + + power_domain = intel_dsc_power_domain(crtc_state); + + wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); + if (!wakeref) + return; + + dss_ctl1 = intel_de_read(dev_priv, dss_ctl1_reg(crtc_state)); + dss_ctl2 = intel_de_read(dev_priv, dss_ctl2_reg(crtc_state)); + + crtc_state->dsc.compression_enable = dss_ctl2 & LEFT_BRANCH_VDSC_ENABLE; + if (!crtc_state->dsc.compression_enable) + goto out; + + crtc_state->dsc.dsc_split = (dss_ctl2 & RIGHT_BRANCH_VDSC_ENABLE) && + (dss_ctl1 & JOINER_ENABLE); + + if (dss_ctl1 & BIG_JOINER_ENABLE) { + crtc_state->bigjoiner = true; + + if (!(dss_ctl1 & MASTER_BIG_JOINER_ENABLE)) { + crtc_state->bigjoiner_slave = true; + if (!WARN_ON(crtc->pipe == PIPE_A)) + crtc_state->bigjoiner_linked_crtc = + intel_get_crtc_for_pipe(dev_priv, crtc->pipe - 1); + } else { + if (!WARN_ON(INTEL_NUM_PIPES(dev_priv) == crtc->pipe + 1)) + crtc_state->bigjoiner_linked_crtc = + intel_get_crtc_for_pipe(dev_priv, crtc->pipe + 1); + } } - dss_ctl1_val = intel_de_read(dev_priv, dss_ctl1_reg); - if (dss_ctl1_val & JOINER_ENABLE) - dss_ctl1_val &= ~JOINER_ENABLE; - intel_de_write(dev_priv, dss_ctl1_reg, dss_ctl1_val); - - dss_ctl2_val = intel_de_read(dev_priv, dss_ctl2_reg); - if (dss_ctl2_val & LEFT_BRANCH_VDSC_ENABLE || - dss_ctl2_val & RIGHT_BRANCH_VDSC_ENABLE) - dss_ctl2_val &= ~(LEFT_BRANCH_VDSC_ENABLE | - RIGHT_BRANCH_VDSC_ENABLE); - intel_de_write(dev_priv, dss_ctl2_reg, dss_ctl2_val); - - /* Disable Power wells for VDSC/joining */ - intel_display_power_put_unchecked(dev_priv, - intel_dsc_power_domain(old_crtc_state)); + + /* FIXME: add more state readout as needed */ + + /* PPS1 */ + if (!is_pipe_dsc(crtc_state)) + val = intel_de_read(dev_priv, DSCA_PICTURE_PARAMETER_SET_1); + else + val = intel_de_read(dev_priv, + ICL_DSC0_PICTURE_PARAMETER_SET_1(pipe)); + vdsc_cfg->bits_per_pixel = val; + crtc_state->dsc.compressed_bpp = vdsc_cfg->bits_per_pixel >> 4; +out: + intel_display_power_put(dev_priv, power_domain, wakeref); } diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.h b/drivers/gpu/drm/i915/display/intel_vdsc.h index e56a3254c214..65d301c23580 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.h +++ b/drivers/gpu/drm/i915/display/intel_vdsc.h @@ -11,15 +11,13 @@ struct intel_encoder; struct intel_crtc_state; -bool intel_dsc_source_support(struct intel_encoder *encoder, - const struct intel_crtc_state *crtc_state); +bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state); void intel_dsc_enable(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); void intel_dsc_disable(const struct intel_crtc_state *crtc_state); int intel_dsc_compute_params(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config); -void intel_dsc_get_config(struct intel_encoder *encoder, - struct intel_crtc_state *crtc_state); +void intel_dsc_get_config(struct intel_crtc_state *crtc_state); enum intel_display_power_domain intel_dsc_power_domain(const struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c index 272cf3ea68d5..44821d94544f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c @@ -202,12 +202,6 @@ retry: if (unlikely(err)) goto out_request; - if (w->ce->engine->emit_init_breadcrumb) { - err = w->ce->engine->emit_init_breadcrumb(rq); - if (unlikely(err)) - goto out_request; - } - /* * w->dma is already exported via (vma|obj)->resv we need only * keep track of the GPU activity within this vma/request, and @@ -217,9 +211,15 @@ retry: if (err) goto out_request; - err = w->ce->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (unlikely(err)) + goto out_request; + } + + err = rq->engine->emit_bb_start(rq, + batch->node.start, batch->node.size, + 0); out_request: if (unlikely(err)) { i915_request_set_error_once(rq, err); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 1904e6e5ea64..b07dc1156a0e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -3097,7 +3097,7 @@ static void retire_requests(struct intel_timeline *tl, struct i915_request *end) break; } -static void eb_request_add(struct i915_execbuffer *eb) +static int eb_request_add(struct i915_execbuffer *eb, int err) { struct i915_request *rq = eb->request; struct intel_timeline * const tl = i915_request_timeline(rq); @@ -3118,6 +3118,7 @@ static void eb_request_add(struct i915_execbuffer *eb) /* Serialise with context_close via the add_to_timeline */ i915_request_set_error_once(rq, -ENOENT); __i915_request_skip(rq); + err = -ENOENT; /* override any transient errors */ } __i915_request_queue(rq, &attr); @@ -3127,6 +3128,8 @@ static void eb_request_add(struct i915_execbuffer *eb) retire_requests(tl, prev); mutex_unlock(&tl->mutex); + + return err; } static const i915_user_extension_fn execbuf_extensions[] = { @@ -3332,7 +3335,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, err = eb_submit(&eb, batch); err_request: i915_request_get(eb.request); - eb_request_add(&eb); + err = eb_request_add(&eb, err); if (eb.fences) signal_fence_array(&eb); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 3389ac972d16..00d24000b5e8 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -82,6 +82,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->mm.madv = I915_MADV_WILLNEED; INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); mutex_init(&obj->mm.get_page.lock); + INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN); + mutex_init(&obj->mm.get_dma_page.lock); if (IS_ENABLED(CONFIG_LOCKDEP) && i915_gem_object_is_shrinkable(obj)) i915_gem_shrinker_taints_mutex(to_i915(obj->base.dev), diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index eaf3d4147be0..be14486f63a7 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -272,8 +272,26 @@ int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, unsigned int tiling, unsigned int stride); struct scatterlist * +__i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + struct i915_gem_object_page_iter *iter, + unsigned int n, + unsigned int *offset); + +static inline struct scatterlist * i915_gem_object_get_sg(struct drm_i915_gem_object *obj, - unsigned int n, unsigned int *offset); + unsigned int n, + unsigned int *offset) +{ + return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset); +} + +static inline struct scatterlist * +i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, + unsigned int n, + unsigned int *offset) +{ + return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset); +} struct page * i915_gem_object_get_page(struct drm_i915_gem_object *obj, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index b5c15557cc87..e2d9b7e1e152 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -56,6 +56,8 @@ struct drm_i915_gem_object_ops { void (*truncate)(struct drm_i915_gem_object *obj); void (*writeback)(struct drm_i915_gem_object *obj); + int (*pread)(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pread *arg); int (*pwrite)(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pwrite *arg); @@ -80,6 +82,14 @@ struct i915_mmap_offset { struct rb_node offset; }; +struct i915_gem_object_page_iter { + struct scatterlist *sg_pos; + unsigned int sg_idx; /* in pages, but 32bit eek! */ + + struct radix_tree_root radix; + struct mutex lock; /* protects this cache */ +}; + struct drm_i915_gem_object { struct drm_gem_object base; @@ -246,13 +256,8 @@ struct drm_i915_gem_object { I915_SELFTEST_DECLARE(unsigned int page_mask); - struct i915_gem_object_page_iter { - struct scatterlist *sg_pos; - unsigned int sg_idx; /* in pages, but 32bit eek! */ - - struct radix_tree_root radix; - struct mutex lock; /* protects this cache */ - } get_page; + struct i915_gem_object_page_iter get_page; + struct i915_gem_object_page_iter get_dma_page; /** * Element within i915->mm.unbound_list or i915->mm.bound_list, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index f60ca6dc911f..e2c7b2a7895f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -33,6 +33,8 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, obj->mm.get_page.sg_pos = pages->sgl; obj->mm.get_page.sg_idx = 0; + obj->mm.get_dma_page.sg_pos = pages->sgl; + obj->mm.get_dma_page.sg_idx = 0; obj->mm.pages = pages; @@ -155,6 +157,8 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) rcu_read_lock(); radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) radix_tree_delete(&obj->mm.get_page.radix, iter.index); + radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0) + radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index); rcu_read_unlock(); } @@ -438,11 +442,12 @@ void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) } struct scatterlist * -i915_gem_object_get_sg(struct drm_i915_gem_object *obj, - unsigned int n, - unsigned int *offset) +__i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + struct i915_gem_object_page_iter *iter, + unsigned int n, + unsigned int *offset) { - struct i915_gem_object_page_iter *iter = &obj->mm.get_page; + const bool dma = iter == &obj->mm.get_dma_page; struct scatterlist *sg; unsigned int idx, count; @@ -471,7 +476,7 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, sg = iter->sg_pos; idx = iter->sg_idx; - count = __sg_page_count(sg); + count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); while (idx + count <= n) { void *entry; @@ -499,7 +504,7 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, idx += count; sg = ____sg_next(sg); - count = __sg_page_count(sg); + count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); } scan: @@ -517,7 +522,7 @@ scan: while (idx + count <= n) { idx += count; sg = ____sg_next(sg); - count = __sg_page_count(sg); + count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); } *offset = n - idx; @@ -584,7 +589,7 @@ i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, struct scatterlist *sg; unsigned int offset; - sg = i915_gem_object_get_sg(obj, n, &offset); + sg = i915_gem_object_get_sg_dma(obj, n, &offset); if (len) *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index 28147aab47b9..3a4dfe2ef1da 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -134,6 +134,58 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, vaddr, dma); } +static int +phys_pwrite(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) +{ + void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; + char __user *user_data = u64_to_user_ptr(args->data_ptr); + int err; + + err = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT); + if (err) + return err; + + /* + * We manually control the domain here and pretend that it + * remains coherent i.e. in the GTT domain, like shmem_pwrite. + */ + i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); + + if (copy_from_user(vaddr, user_data, args->size)) + return -EFAULT; + + drm_clflush_virt_range(vaddr, args->size); + intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); + + i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); + return 0; +} + +static int +phys_pread(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pread *args) +{ + void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; + char __user *user_data = u64_to_user_ptr(args->data_ptr); + int err; + + err = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); + if (err) + return err; + + drm_clflush_virt_range(vaddr, args->size); + if (copy_to_user(user_data, vaddr, args->size)) + return -EFAULT; + + return 0; +} + static void phys_release(struct drm_i915_gem_object *obj) { fput(obj->base.filp); @@ -144,6 +196,9 @@ static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { .get_pages = i915_gem_object_get_pages_phys, .put_pages = i915_gem_object_put_pages_phys, + .pread = phys_pread, + .pwrite = phys_pwrite, + .release = phys_release, }; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 84b2707d8b17..29bffc6afcc1 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -497,6 +497,43 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) return 0; } +static void dbg_poison(struct i915_ggtt *ggtt, + dma_addr_t addr, resource_size_t size, + u8 x) +{ +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) + if (!drm_mm_node_allocated(&ggtt->error_capture)) + return; + + if (ggtt->vm.bind_async_flags & I915_VMA_GLOBAL_BIND) + return; /* beware stop_machine() inversion */ + + GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); + + mutex_lock(&ggtt->error_mutex); + while (size) { + void __iomem *s; + + ggtt->vm.insert_page(&ggtt->vm, addr, + ggtt->error_capture.start, + I915_CACHE_NONE, 0); + mb(); + + s = io_mapping_map_wc(&ggtt->iomap, + ggtt->error_capture.start, + PAGE_SIZE); + memset_io(s, x, PAGE_SIZE); + io_mapping_unmap(s); + + addr += PAGE_SIZE; + size -= PAGE_SIZE; + } + mb(); + ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE); + mutex_unlock(&ggtt->error_mutex); +#endif +} + static struct sg_table * i915_pages_create_for_stolen(struct drm_device *dev, resource_size_t offset, resource_size_t size) @@ -540,6 +577,11 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) if (IS_ERR(pages)) return PTR_ERR(pages); + dbg_poison(&to_i915(obj->base.dev)->ggtt, + sg_dma_address(pages->sgl), + sg_dma_len(pages->sgl), + POISON_INUSE); + __i915_gem_object_set_pages(obj, pages, obj->stolen->size); return 0; @@ -549,6 +591,12 @@ static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, struct sg_table *pages) { /* Should only be called from i915_gem_object_release_stolen() */ + + dbg_poison(&to_i915(obj->base.dev)->ggtt, + sg_dma_address(pages->sgl), + sg_dma_len(pages->sgl), + POISON_FREE); + sg_free_table(pages); kfree(pages); } diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index c30adc05fa98..680bd9442eb0 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -131,17 +131,17 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt)); do { - GEM_BUG_ON(iter.sg->length < I915_GTT_PAGE_SIZE); + GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE); vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma); iter.dma += I915_GTT_PAGE_SIZE; if (iter.dma == iter.max) { iter.sg = __sg_next(iter.sg); - if (!iter.sg) + if (!iter.sg || sg_dma_len(iter.sg) == 0) break; iter.dma = sg_dma_address(iter.sg); - iter.max = iter.dma + iter.sg->length; + iter.max = iter.dma + sg_dma_len(iter.sg); } if (++act_pte == GEN6_PTES) { diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 38c7069b7749..a37c968ef8f7 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -372,19 +372,19 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2)); vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); do { - GEM_BUG_ON(iter->sg->length < I915_GTT_PAGE_SIZE); + GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; iter->dma += I915_GTT_PAGE_SIZE; if (iter->dma >= iter->max) { iter->sg = __sg_next(iter->sg); - if (!iter->sg) { + if (!iter->sg || sg_dma_len(iter->sg) == 0) { idx = 0; break; } iter->dma = sg_dma_address(iter->sg); - iter->max = iter->dma + iter->sg->length; + iter->max = iter->dma + sg_dma_len(iter->sg); } if (gen8_pd_index(++idx, 0) == 0) { @@ -413,8 +413,8 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, u32 flags) { const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags); + unsigned int rem = sg_dma_len(iter->sg); u64 start = vma->node.start; - dma_addr_t rem = iter->sg->length; GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm)); @@ -456,7 +456,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } do { - GEM_BUG_ON(iter->sg->length < page_size); + GEM_BUG_ON(sg_dma_len(iter->sg) < page_size); vaddr[index++] = encode | iter->dma; start += page_size; @@ -467,7 +467,10 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, if (!iter->sg) break; - rem = iter->sg->length; + rem = sg_dma_len(iter->sg); + if (!rem) + break; + iter->dma = sg_dma_address(iter->sg); iter->max = iter->dma + rem; @@ -525,7 +528,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, } vma->page_sizes.gtt |= page_size; - } while (iter->sg); + } while (iter->sg && sg_dma_len(iter->sg)); } static void gen8_ppgtt_insert(struct i915_address_space *vm, diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c index d8b206e53660..a24cc1ff08a0 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c @@ -30,18 +30,21 @@ #include "i915_trace.h" #include "intel_breadcrumbs.h" #include "intel_context.h" +#include "intel_engine_pm.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" -static void irq_enable(struct intel_engine_cs *engine) +static bool irq_enable(struct intel_engine_cs *engine) { if (!engine->irq_enable) - return; + return false; /* Caller disables interrupts */ spin_lock(&engine->gt->irq_lock); engine->irq_enable(engine); spin_unlock(&engine->gt->irq_lock); + + return true; } static void irq_disable(struct intel_engine_cs *engine) @@ -57,12 +60,11 @@ static void irq_disable(struct intel_engine_cs *engine) static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) { - lockdep_assert_held(&b->irq_lock); - - if (!b->irq_engine || b->irq_armed) - return; - - if (!intel_gt_pm_get_if_awake(b->irq_engine->gt)) + /* + * Since we are waiting on a request, the GPU should be busy + * and should have its own rpm reference. + */ + if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt))) return; /* @@ -73,25 +75,24 @@ static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) */ WRITE_ONCE(b->irq_armed, true); - /* - * Since we are waiting on a request, the GPU should be busy - * and should have its own rpm reference. This is tracked - * by i915->gt.awake, we can forgo holding our own wakref - * for the interrupt as before i915->gt.awake is released (when - * the driver is idle) we disarm the breadcrumbs. - */ - - if (!b->irq_enabled++) - irq_enable(b->irq_engine); + /* Requests may have completed before we could enable the interrupt. */ + if (!b->irq_enabled++ && irq_enable(b->irq_engine)) + irq_work_queue(&b->irq_work); } -static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) +static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) { - lockdep_assert_held(&b->irq_lock); - - if (!b->irq_engine || !b->irq_armed) + if (!b->irq_engine) return; + spin_lock(&b->irq_lock); + if (!b->irq_armed) + __intel_breadcrumbs_arm_irq(b); + spin_unlock(&b->irq_lock); +} + +static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) +{ GEM_BUG_ON(!b->irq_enabled); if (!--b->irq_enabled) irq_disable(b->irq_engine); @@ -100,20 +101,37 @@ static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) intel_gt_pm_put_async(b->irq_engine->gt); } +static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) +{ + spin_lock(&b->irq_lock); + if (b->irq_armed) + __intel_breadcrumbs_disarm_irq(b); + spin_unlock(&b->irq_lock); +} + static void add_signaling_context(struct intel_breadcrumbs *b, struct intel_context *ce) { - intel_context_get(ce); - list_add_tail(&ce->signal_link, &b->signalers); - if (list_is_first(&ce->signal_link, &b->signalers)) - __intel_breadcrumbs_arm_irq(b); + lockdep_assert_held(&ce->signal_lock); + + spin_lock(&b->signalers_lock); + list_add_rcu(&ce->signal_link, &b->signalers); + spin_unlock(&b->signalers_lock); } -static void remove_signaling_context(struct intel_breadcrumbs *b, +static bool remove_signaling_context(struct intel_breadcrumbs *b, struct intel_context *ce) { - list_del(&ce->signal_link); - intel_context_put(ce); + lockdep_assert_held(&ce->signal_lock); + + if (!list_empty(&ce->signals)) + return false; + + spin_lock(&b->signalers_lock); + list_del_rcu(&ce->signal_link); + spin_unlock(&b->signalers_lock); + + return true; } static inline bool __request_completed(const struct i915_request *rq) @@ -174,73 +192,103 @@ static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) intel_engine_add_retire(b->irq_engine, tl); } -static bool __signal_request(struct i915_request *rq, struct list_head *signals) +static bool __signal_request(struct i915_request *rq) { - clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); if (!__dma_fence_signal(&rq->fence)) { i915_request_put(rq); return false; } - list_add_tail(&rq->signal_link, signals); return true; } +static struct llist_node * +slist_add(struct llist_node *node, struct llist_node *head) +{ + node->next = head; + return node; +} + static void signal_irq_work(struct irq_work *work) { struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); const ktime_t timestamp = ktime_get(); - struct intel_context *ce, *cn; - struct list_head *pos, *next; - LIST_HEAD(signal); - - spin_lock(&b->irq_lock); + struct llist_node *signal, *sn; + struct intel_context *ce; - if (list_empty(&b->signalers)) - __intel_breadcrumbs_disarm_irq(b); + signal = NULL; + if (unlikely(!llist_empty(&b->signaled_requests))) + signal = llist_del_all(&b->signaled_requests); - list_splice_init(&b->signaled_requests, &signal); + /* + * Keep the irq armed until the interrupt after all listeners are gone. + * + * Enabling/disabling the interrupt is rather costly, roughly a couple + * of hundred microseconds. If we are proactive and enable/disable + * the interrupt around every request that wants a breadcrumb, we + * quickly drown in the extra orders of magnitude of latency imposed + * on request submission. + * + * So we try to be lazy, and keep the interrupts enabled until no + * more listeners appear within a breadcrumb interrupt interval (that + * is until a request completes that no one cares about). The + * observation is that listeners come in batches, and will often + * listen to a bunch of requests in succession. Though note on icl+, + * interrupts are always enabled due to concerns with rc6 being + * dysfunctional with per-engine interrupt masking. + * + * We also try to avoid raising too many interrupts, as they may + * be generated by userspace batches and it is unfortunately rather + * too easy to drown the CPU under a flood of GPU interrupts. Thus + * whenever no one appears to be listening, we turn off the interrupts. + * Fewer interrupts should conserve power -- at the very least, fewer + * interrupt draw less ire from other users of the system and tools + * like powertop. + */ + if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers)) + intel_breadcrumbs_disarm_irq(b); - list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { - GEM_BUG_ON(list_empty(&ce->signals)); + rcu_read_lock(); + list_for_each_entry_rcu(ce, &b->signalers, signal_link) { + struct i915_request *rq; - list_for_each_safe(pos, next, &ce->signals) { - struct i915_request *rq = - list_entry(pos, typeof(*rq), signal_link); + list_for_each_entry_rcu(rq, &ce->signals, signal_link) { + bool release; - GEM_BUG_ON(!check_signal_order(ce, rq)); if (!__request_completed(rq)) break; + if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, + &rq->fence.flags)) + break; + /* * Queue for execution after dropping the signaling * spinlock as the callback chain may end up adding * more signalers to the same context or engine. */ - __signal_request(rq, &signal); - } + spin_lock(&ce->signal_lock); + list_del_rcu(&rq->signal_link); + release = remove_signaling_context(b, ce); + spin_unlock(&ce->signal_lock); - /* - * We process the list deletion in bulk, only using a list_add - * (not list_move) above but keeping the status of - * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. - */ - if (!list_is_first(pos, &ce->signals)) { - /* Advance the list to the first incomplete request */ - __list_del_many(&ce->signals, pos); - if (&ce->signals == pos) { /* now empty */ + if (__signal_request(rq)) + /* We own signal_node now, xfer to local list */ + signal = slist_add(&rq->signal_node, signal); + + if (release) { add_retire(b, ce->timeline); - remove_signaling_context(b, ce); + intel_context_put(ce); } } } + rcu_read_unlock(); - spin_unlock(&b->irq_lock); - - list_for_each_safe(pos, next, &signal) { + llist_for_each_safe(signal, sn, signal) { struct i915_request *rq = - list_entry(pos, typeof(*rq), signal_link); + llist_entry(signal, typeof(*rq), signal_node); struct list_head cb_list; spin_lock(&rq->lock); @@ -251,6 +299,9 @@ static void signal_irq_work(struct irq_work *work) i915_request_put(rq); } + + if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers)) + intel_breadcrumbs_arm_irq(b); } struct intel_breadcrumbs * @@ -262,14 +313,15 @@ intel_breadcrumbs_create(struct intel_engine_cs *irq_engine) if (!b) return NULL; - spin_lock_init(&b->irq_lock); + b->irq_engine = irq_engine; + + spin_lock_init(&b->signalers_lock); INIT_LIST_HEAD(&b->signalers); - INIT_LIST_HEAD(&b->signaled_requests); + init_llist_head(&b->signaled_requests); + spin_lock_init(&b->irq_lock); init_irq_work(&b->irq_work, signal_irq_work); - b->irq_engine = irq_engine; - return b; } @@ -292,27 +344,28 @@ void intel_breadcrumbs_reset(struct intel_breadcrumbs *b) void intel_breadcrumbs_park(struct intel_breadcrumbs *b) { - unsigned long flags; - - if (!READ_ONCE(b->irq_armed)) - return; - - spin_lock_irqsave(&b->irq_lock, flags); - __intel_breadcrumbs_disarm_irq(b); - spin_unlock_irqrestore(&b->irq_lock, flags); - - if (!list_empty(&b->signalers)) - irq_work_queue(&b->irq_work); + /* Kick the work once more to drain the signalers */ + irq_work_sync(&b->irq_work); + while (unlikely(READ_ONCE(b->irq_armed))) { + local_irq_disable(); + signal_irq_work(&b->irq_work); + local_irq_enable(); + cond_resched(); + } + GEM_BUG_ON(!list_empty(&b->signalers)); } void intel_breadcrumbs_free(struct intel_breadcrumbs *b) { + irq_work_sync(&b->irq_work); + GEM_BUG_ON(!list_empty(&b->signalers)); + GEM_BUG_ON(b->irq_armed); kfree(b); } -static void insert_breadcrumb(struct i915_request *rq, - struct intel_breadcrumbs *b) +static void insert_breadcrumb(struct i915_request *rq) { + struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs; struct intel_context *ce = rq->context; struct list_head *pos; @@ -327,12 +380,14 @@ static void insert_breadcrumb(struct i915_request *rq, * its signal completion. */ if (__request_completed(rq)) { - if (__signal_request(rq, &b->signaled_requests)) + if (__signal_request(rq) && + llist_add(&rq->signal_node, &b->signaled_requests)) irq_work_queue(&b->irq_work); return; } if (list_empty(&ce->signals)) { + intel_context_get(ce); add_signaling_context(b, ce); pos = &ce->signals; } else { @@ -358,18 +413,22 @@ static void insert_breadcrumb(struct i915_request *rq, break; } } - list_add(&rq->signal_link, pos); + list_add_rcu(&rq->signal_link, pos); GEM_BUG_ON(!check_signal_order(ce, rq)); + GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)); set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); - /* Check after attaching to irq, interrupt may have already fired. */ - if (__request_completed(rq)) - irq_work_queue(&b->irq_work); + /* + * Defer enabling the interrupt to after HW submission and recheck + * the request as it may have completed and raised the interrupt as + * we were attaching it into the lists. + */ + irq_work_queue(&b->irq_work); } bool i915_request_enable_breadcrumb(struct i915_request *rq) { - struct intel_breadcrumbs *b; + struct intel_context *ce = rq->context; /* Serialises with i915_request_retire() using rq->lock */ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) @@ -384,67 +443,30 @@ bool i915_request_enable_breadcrumb(struct i915_request *rq) if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) return true; - /* - * rq->engine is locked by rq->engine->active.lock. That however - * is not known until after rq->engine has been dereferenced and - * the lock acquired. Hence we acquire the lock and then validate - * that rq->engine still matches the lock we hold for it. - * - * Here, we are using the breadcrumb lock as a proxy for the - * rq->engine->active.lock, and we know that since the breadcrumb - * will be serialised within i915_request_submit/i915_request_unsubmit, - * the engine cannot change while active as long as we hold the - * breadcrumb lock on that engine. - * - * From the dma_fence_enable_signaling() path, we are outside of the - * request submit/unsubmit path, and so we must be more careful to - * acquire the right lock. - */ - b = READ_ONCE(rq->engine)->breadcrumbs; - spin_lock(&b->irq_lock); - while (unlikely(b != READ_ONCE(rq->engine)->breadcrumbs)) { - spin_unlock(&b->irq_lock); - b = READ_ONCE(rq->engine)->breadcrumbs; - spin_lock(&b->irq_lock); - } - - /* - * Now that we are finally serialised with request submit/unsubmit, - * [with b->irq_lock] and with i915_request_retire() [via checking - * SIGNALED with rq->lock] confirm the request is indeed active. If - * it is no longer active, the breadcrumb will be attached upon - * i915_request_submit(). - */ + spin_lock(&ce->signal_lock); if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) - insert_breadcrumb(rq, b); - - spin_unlock(&b->irq_lock); + insert_breadcrumb(rq); + spin_unlock(&ce->signal_lock); return true; } void i915_request_cancel_breadcrumb(struct i915_request *rq) { - struct intel_breadcrumbs *b = rq->engine->breadcrumbs; + struct intel_context *ce = rq->context; + bool release; - /* - * We must wait for b->irq_lock so that we know the interrupt handler - * has released its reference to the intel_context and has completed - * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if - * required). - */ - spin_lock(&b->irq_lock); - if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { - struct intel_context *ce = rq->context; + if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) + return; - list_del(&rq->signal_link); - if (list_empty(&ce->signals)) - remove_signaling_context(b, ce); + spin_lock(&ce->signal_lock); + list_del_rcu(&rq->signal_link); + release = remove_signaling_context(rq->engine->breadcrumbs, ce); + spin_unlock(&ce->signal_lock); + if (release) + intel_context_put(ce); - clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); - i915_request_put(rq); - } - spin_unlock(&b->irq_lock); + i915_request_put(rq); } static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p) @@ -454,18 +476,17 @@ static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p) drm_printf(p, "Signals:\n"); - spin_lock_irq(&b->irq_lock); - list_for_each_entry(ce, &b->signalers, signal_link) { - list_for_each_entry(rq, &ce->signals, signal_link) { + rcu_read_lock(); + list_for_each_entry_rcu(ce, &b->signalers, signal_link) { + list_for_each_entry_rcu(rq, &ce->signals, signal_link) drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", rq->fence.context, rq->fence.seqno, i915_request_completed(rq) ? "!" : i915_request_started(rq) ? "*" : "", jiffies_to_msecs(jiffies - rq->emitted_jiffies)); - } } - spin_unlock_irq(&b->irq_lock); + rcu_read_unlock(); } void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h index 8e53b9942695..a74bb3062bd8 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs_types.h @@ -29,18 +29,16 @@ * the overhead of waking that client is much preferred. */ struct intel_breadcrumbs { - spinlock_t irq_lock; /* protects the lists used in hardirq context */ - /* Not all breadcrumbs are attached to physical HW */ struct intel_engine_cs *irq_engine; + spinlock_t signalers_lock; /* protects the list of signalers */ struct list_head signalers; - struct list_head signaled_requests; + struct llist_head signaled_requests; + spinlock_t irq_lock; /* protects the interrupt from hardirq context */ struct irq_work irq_work; /* for use from inside irq_lock */ - unsigned int irq_enabled; - bool irq_armed; }; diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 92a3f25c4006..349e7fa1488d 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -25,11 +25,18 @@ static struct intel_context *intel_context_alloc(void) return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); } -void intel_context_free(struct intel_context *ce) +static void rcu_context_free(struct rcu_head *rcu) { + struct intel_context *ce = container_of(rcu, typeof(*ce), rcu); + kmem_cache_free(global.slab_ce, ce); } +void intel_context_free(struct intel_context *ce) +{ + call_rcu(&ce->rcu, rcu_context_free); +} + struct intel_context * intel_context_create(struct intel_engine_cs *engine) { @@ -356,8 +363,7 @@ static int __intel_context_active(struct i915_active *active) } void -intel_context_init(struct intel_context *ce, - struct intel_engine_cs *engine) +intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine) { GEM_BUG_ON(!engine->cops); GEM_BUG_ON(!engine->gt->vm); @@ -373,7 +379,8 @@ intel_context_init(struct intel_context *ce, ce->vm = i915_vm_get(engine->gt->vm); - INIT_LIST_HEAD(&ce->signal_link); + /* NB ce->signal_link/lock is used under RCU */ + spin_lock_init(&ce->signal_lock); INIT_LIST_HEAD(&ce->signals); mutex_init(&ce->pin_mutex); diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index 552cb57a2e8c..52fa9c132746 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -25,6 +25,7 @@ DECLARE_EWMA(runtime, 3, 8); struct i915_gem_context; struct i915_gem_ww_ctx; struct i915_vma; +struct intel_breadcrumbs; struct intel_context; struct intel_ring; @@ -44,7 +45,16 @@ struct intel_context_ops { }; struct intel_context { - struct kref ref; + /* + * Note: Some fields may be accessed under RCU. + * + * Unless otherwise noted a field can safely be assumed to be protected + * by strong reference counting. + */ + union { + struct kref ref; /* no kref_get_unless_zero()! */ + struct rcu_head rcu; + }; struct intel_engine_cs *engine; struct intel_engine_cs *inflight; @@ -54,8 +64,15 @@ struct intel_context { struct i915_address_space *vm; struct i915_gem_context __rcu *gem_context; - struct list_head signal_link; - struct list_head signals; + /* + * @signal_lock protects the list of requests that need signaling, + * @signals. While there are any requests that need signaling, + * we add the context to the breadcrumbs worker, and remove it + * upon completion/cancellation of the last request. + */ + struct list_head signal_link; /* Accessed under RCU */ + struct list_head signals; /* Guarded by signal_lock */ + spinlock_t signal_lock; /* protects signals, the list of requests */ struct i915_vma *state; struct intel_ring *ring; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 5bfb5f7ed02c..0b31670343f5 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -305,8 +305,9 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->i915 = i915; engine->gt = gt; engine->uncore = gt->uncore; - engine->hw_id = engine->guc_id = info->hw_id; engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); + engine->hw_id = info->hw_id; + engine->guc_id = MAKE_GUC_ID(info->class, info->instance); engine->class = info->class; engine->instance = info->instance; @@ -371,7 +372,8 @@ static void __setup_engine_capabilities(struct intel_engine_cs *engine) * instances. */ if ((INTEL_GEN(i915) >= 11 && - engine->gt->info.vdbox_sfc_access & engine->mask) || + (engine->gt->info.vdbox_sfc_access & + BIT(engine->instance))) || (INTEL_GEN(i915) >= 9 && engine->instance == 0)) engine->uabi_capabilities |= I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; @@ -1599,6 +1601,41 @@ static unsigned long list_count(struct list_head *list) return count; } +static unsigned long read_ul(void *p, size_t x) +{ + return *(unsigned long *)(p + x); +} + +static void print_properties(struct intel_engine_cs *engine, + struct drm_printer *m) +{ + static const struct pmap { + size_t offset; + const char *name; + } props[] = { +#define P(x) { \ + .offset = offsetof(typeof(engine->props), x), \ + .name = #x \ +} + P(heartbeat_interval_ms), + P(max_busywait_duration_ns), + P(preempt_timeout_ms), + P(stop_timeout_ms), + P(timeslice_duration_ms), + + {}, +#undef P + }; + const struct pmap *p; + + drm_printf(m, "\tProperties:\n"); + for (p = props; p->name; p++) + drm_printf(m, "\t\t%s: %lu [default %lu]\n", + p->name, + read_ul(&engine->props, p->offset), + read_ul(&engine->defaults, p->offset)); +} + void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m, const char *header, ...) @@ -1641,6 +1678,7 @@ void intel_engine_dump(struct intel_engine_cs *engine, drm_printf(m, "\tReset count: %d (global %d)\n", i915_reset_engine_count(error, engine), i915_reset_count(error)); + print_properties(engine, m); drm_printf(m, "\tRequests:\n"); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index 5067d0524d4b..9060385cd69e 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -41,6 +41,8 @@ static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq) { engine->wakeref_serial = READ_ONCE(engine->serial) + 1; i915_request_add_active_barriers(rq); + if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine)) + engine->heartbeat.systole = i915_request_get(rq); } static void show_heartbeat(const struct i915_request *rq, @@ -144,8 +146,6 @@ static void heartbeat(struct work_struct *wrk) goto unlock; idle_pulse(engine, rq); - if (engine->i915->params.enable_hangcheck) - engine->heartbeat.systole = i915_request_get(rq); __i915_request_commit(rq); __i915_request_queue(rq, &attr); @@ -153,7 +153,7 @@ static void heartbeat(struct work_struct *wrk) unlock: mutex_unlock(&ce->timeline->mutex); out: - if (!next_heartbeat(engine)) + if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine)) i915_request_put(fetch_and_zero(&engine->heartbeat.systole)); intel_engine_pm_put(engine); } diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index f7b2e07e2229..499b09cb4acf 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -17,6 +17,25 @@ #include "intel_ring.h" #include "shmem_utils.h" +static void dbg_poison_ce(struct intel_context *ce) +{ + if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + return; + + if (ce->state) { + struct drm_i915_gem_object *obj = ce->state->obj; + int type = i915_coherent_map_type(ce->engine->i915); + void *map; + + map = i915_gem_object_pin_map(obj, type); + if (!IS_ERR(map)) { + memset(map, CONTEXT_REDZONE, obj->base.size); + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); + } + } +} + static int __engine_unpark(struct intel_wakeref *wf) { struct intel_engine_cs *engine = @@ -32,20 +51,14 @@ static int __engine_unpark(struct intel_wakeref *wf) if (ce) { GEM_BUG_ON(test_bit(CONTEXT_VALID_BIT, &ce->flags)); + /* Flush all pending HW writes before we touch the context */ + while (unlikely(intel_context_inflight(ce))) + intel_engine_flush_submission(engine); + /* First poison the image to verify we never fully trust it */ - if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && ce->state) { - struct drm_i915_gem_object *obj = ce->state->obj; - int type = i915_coherent_map_type(engine->i915); - void *map; - - map = i915_gem_object_pin_map(obj, type); - if (!IS_ERR(map)) { - memset(map, CONTEXT_REDZONE, obj->base.size); - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - } - } + dbg_poison_ce(ce); + /* Scrub the context image after our loss of control */ ce->ops->reset(ce); } diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 188a5f70177d..cf94525be2c1 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -1383,7 +1383,7 @@ intel_partial_pages(const struct i915_ggtt_view *view, if (ret) goto err_sg_alloc; - iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); + iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset); GEM_BUG_ON(!iter); sg = st->sgl; @@ -1391,7 +1391,7 @@ intel_partial_pages(const struct i915_ggtt_view *view, do { unsigned int len; - len = min(iter->length - (offset << PAGE_SHIFT), + len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), count << PAGE_SHIFT); sg_set_page(sg, NULL, len, 0); sg_dma_address(sg) = diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 39b428c5049c..44f1d51e5ae5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -614,6 +614,8 @@ void intel_gt_driver_remove(struct intel_gt *gt) void intel_gt_driver_unregister(struct intel_gt *gt) { + intel_wakeref_t wakeref; + intel_rps_driver_unregister(>->rps); /* @@ -622,16 +624,15 @@ void intel_gt_driver_unregister(struct intel_gt *gt) * resources. */ intel_gt_set_wedged(gt); + + /* Scrub all HW state upon release */ + with_intel_runtime_pm(gt->uncore->rpm, wakeref) + __intel_gt_reset(gt, ALL_ENGINES); } void intel_gt_driver_release(struct intel_gt *gt) { struct i915_address_space *vm; - intel_wakeref_t wakeref; - - /* Scrub all HW state upon release */ - with_intel_runtime_pm(gt->uncore->rpm, wakeref) - __intel_gt_reset(gt, ALL_ENGINES); vm = fetch_and_zero(>->vm); if (vm) /* FIXME being called twice on error paths :( */ diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index 3f1114b58b01..7bfe9072be9a 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -324,7 +324,7 @@ static void cnl_setup_private_ppat(struct intel_uncore *uncore) GEN8_PPAT_WC | GEN8_PPAT_LLCELLC); intel_uncore_write(uncore, GEN10_PAT_INDEX(2), - GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); + GEN8_PPAT_WB | GEN8_PPAT_ELLC_OVERRIDE); intel_uncore_write(uncore, GEN10_PAT_INDEX(3), GEN8_PPAT_UC); @@ -349,17 +349,23 @@ static void cnl_setup_private_ppat(struct intel_uncore *uncore) */ static void bdw_setup_private_ppat(struct intel_uncore *uncore) { + struct drm_i915_private *i915 = uncore->i915; u64 pat; pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */ - GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */ GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */ GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) | GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) | GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) | GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3)); + /* for scanout with eLLC */ + if (INTEL_GEN(i915) >= 9) + pat |= GEN8_PPAT(2, GEN8_PPAT_WB | GEN8_PPAT_ELLC_OVERRIDE); + else + pat |= GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC); + intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat)); intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat)); } diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index c13c650ced22..8a33940a71f3 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -580,7 +580,7 @@ static inline struct sgt_dma { struct scatterlist *sg = vma->pages->sgl; dma_addr_t addr = sg_dma_address(sg); - return (struct sgt_dma){ sg, addr, addr + sg->length }; + return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) }; } #endif diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index f82c6dd1de18..7614a3d24fca 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -182,6 +182,7 @@ struct virtual_engine { struct intel_engine_cs base; struct intel_context context; + struct rcu_work rcu; /* * We allow only a single request through the virtual engine at a time @@ -1215,7 +1216,8 @@ static void intel_engine_context_out(struct intel_engine_cs *engine) static void execlists_check_context(const struct intel_context *ce, - const struct intel_engine_cs *engine) + const struct intel_engine_cs *engine, + const char *when) { const struct intel_ring *ring = ce->ring; u32 *regs = ce->lrc_reg_state; @@ -1250,7 +1252,7 @@ execlists_check_context(const struct intel_context *ce, valid = false; } - WARN_ONCE(!valid, "Invalid lrc state found before submission\n"); + WARN_ONCE(!valid, "Invalid lrc state found %s submission\n", when); } static void restore_default_state(struct intel_context *ce, @@ -1346,7 +1348,7 @@ __execlists_schedule_in(struct i915_request *rq) reset_active(rq, engine); if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) - execlists_check_context(ce, engine); + execlists_check_context(ce, engine, "before"); if (ce->tag) { /* Use a fixed tag for OA and friends */ @@ -1417,6 +1419,9 @@ __execlists_schedule_out(struct i915_request *rq, * refrain from doing non-trivial work here. */ + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + execlists_check_context(ce, engine, "after"); + /* * If we have just completed this context, the engine may now be * idle and we want to re-enter powersaving. @@ -2495,25 +2500,11 @@ invalidate_csb_entries(const u64 *first, const u64 *last) * bits 47-57: sw context id of the lrc the GT switched away from * bits 58-63: sw counter of the lrc the GT switched away from */ -static inline bool gen12_csb_parse(const u64 *csb) +static inline bool gen12_csb_parse(const u64 csb) { - bool ctx_away_valid; - bool new_queue; - u64 entry; - - /* HSD#22011248461 */ - entry = READ_ONCE(*csb); - if (unlikely(entry == -1)) { - preempt_disable(); - if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 50)) - GEM_WARN_ON("50us CSB timeout"); - preempt_enable(); - } - WRITE_ONCE(*(u64 *)csb, -1); - - ctx_away_valid = GEN12_CSB_CTX_VALID(upper_32_bits(entry)); - new_queue = - lower_32_bits(entry) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE; + bool ctx_away_valid = GEN12_CSB_CTX_VALID(upper_32_bits(csb)); + bool new_queue = + lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE; /* * The context switch detail is not guaranteed to be 5 when a preemption @@ -2523,7 +2514,7 @@ static inline bool gen12_csb_parse(const u64 *csb) * would require some extra handling, but we don't support that. */ if (!ctx_away_valid || new_queue) { - GEM_BUG_ON(!GEN12_CSB_CTX_VALID(lower_32_bits(entry))); + GEM_BUG_ON(!GEN12_CSB_CTX_VALID(lower_32_bits(csb))); return true; } @@ -2532,19 +2523,79 @@ static inline bool gen12_csb_parse(const u64 *csb) * context switch on an unsuccessful wait instruction since we always * use polling mode. */ - GEM_BUG_ON(GEN12_CTX_SWITCH_DETAIL(upper_32_bits(entry))); + GEM_BUG_ON(GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb))); return false; } -static inline bool gen8_csb_parse(const u64 *csb) +static inline bool gen8_csb_parse(const u64 csb) { - return *csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED); + return csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED); +} + +static noinline u64 +wa_csb_read(const struct intel_engine_cs *engine, u64 * const csb) +{ + u64 entry; + + /* + * Reading from the HWSP has one particular advantage: we can detect + * a stale entry. Since the write into HWSP is broken, we have no reason + * to trust the HW at all, the mmio entry may equally be unordered, so + * we prefer the path that is self-checking and as a last resort, + * return the mmio value. + * + * tgl,dg1:HSDES#22011327657 + */ + preempt_disable(); + if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 10)) { + int idx = csb - engine->execlists.csb_status; + int status; + + status = GEN8_EXECLISTS_STATUS_BUF; + if (idx >= 6) { + status = GEN11_EXECLISTS_STATUS_BUF2; + idx -= 6; + } + status += sizeof(u64) * idx; + + entry = intel_uncore_read64(engine->uncore, + _MMIO(engine->mmio_base + status)); + } + preempt_enable(); + + return entry; +} + +static inline u64 +csb_read(const struct intel_engine_cs *engine, u64 * const csb) +{ + u64 entry = READ_ONCE(*csb); + + /* + * Unfortunately, the GPU does not always serialise its write + * of the CSB entries before its write of the CSB pointer, at least + * from the perspective of the CPU, using what is known as a Global + * Observation Point. We may read a new CSB tail pointer, but then + * read the stale CSB entries, causing us to misinterpret the + * context-switch events, and eventually declare the GPU hung. + * + * icl:HSDES#1806554093 + * tgl:HSDES#22011248461 + */ + if (unlikely(entry == -1)) + entry = wa_csb_read(engine, csb); + + /* Consume this entry so that we can spot its future reuse. */ + WRITE_ONCE(*csb, -1); + + /* ELSP is an implicit wmb() before the GPU wraps and overwrites csb */ + return entry; } static void process_csb(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; - const u64 * const buf = execlists->csb_status; + u64 * const buf = execlists->csb_status; const u8 num_entries = execlists->csb_size; u8 head, tail; @@ -2602,6 +2653,7 @@ static void process_csb(struct intel_engine_cs *engine) rmb(); do { bool promote; + u64 csb; if (++head == num_entries) head = 0; @@ -2624,15 +2676,14 @@ static void process_csb(struct intel_engine_cs *engine) * status notifier. */ + csb = csb_read(engine, buf + head); ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n", - head, - upper_32_bits(buf[head]), - lower_32_bits(buf[head])); + head, upper_32_bits(csb), lower_32_bits(csb)); if (INTEL_GEN(engine->i915) >= 12) - promote = gen12_csb_parse(buf + head); + promote = gen12_csb_parse(csb); else - promote = gen8_csb_parse(buf + head); + promote = gen8_csb_parse(csb); if (promote) { struct i915_request * const *old = execlists->active; @@ -2787,6 +2838,9 @@ static void __execlists_hold(struct i915_request *rq) static bool execlists_hold(struct intel_engine_cs *engine, struct i915_request *rq) { + if (i915_request_on_hold(rq)) + return false; + spin_lock_irq(&engine->active.lock); if (i915_request_completed(rq)) { /* too late! */ @@ -2987,6 +3041,8 @@ static struct execlists_capture *capture_regs(struct intel_engine_cs *engine) if (!cap->error->gt->engine) goto err_gt; + cap->error->gt->engine->hung = true; + return cap; err_gt: @@ -3168,8 +3224,10 @@ static void execlists_submission_tasklet(unsigned long data) spin_unlock_irqrestore(&engine->active.lock, flags); /* Recheck after serialising with direct-submission */ - if (unlikely(timeout && preempt_timeout(engine))) + if (unlikely(timeout && preempt_timeout(engine))) { + cancel_timer(&engine->execlists.preempt); execlists_reset(engine, "preemption time out"); + } } } @@ -4047,6 +4105,8 @@ static void reset_csb_pointers(struct intel_engine_cs *engine) static void execlists_sanitize(struct intel_engine_cs *engine) { + GEM_BUG_ON(execlists_active(&engine->execlists)); + /* * Poison residual state on resume, in case the suspend didn't! * @@ -4376,6 +4436,7 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) /* Mark all executing requests as skipped. */ list_for_each_entry(rq, &engine->active.requests, sched.link) mark_eio(rq); + intel_engine_signal_breadcrumbs(engine); /* Flush the queued requests to the timeline list (for retiring). */ while ((rb = rb_first_cached(&execlists->queue))) { @@ -5425,44 +5486,90 @@ static struct list_head *virtual_queue(struct virtual_engine *ve) return &ve->base.execlists.default_priolist.requests[0]; } -static void virtual_context_destroy(struct kref *kref) +static void rcu_virtual_context_destroy(struct work_struct *wrk) { struct virtual_engine *ve = - container_of(kref, typeof(*ve), context.ref); + container_of(wrk, typeof(*ve), rcu.work); unsigned int n; - GEM_BUG_ON(!list_empty(virtual_queue(ve))); - GEM_BUG_ON(ve->request); GEM_BUG_ON(ve->context.inflight); + /* Preempt-to-busy may leave a stale request behind. */ + if (unlikely(ve->request)) { + struct i915_request *old; + + spin_lock_irq(&ve->base.active.lock); + + old = fetch_and_zero(&ve->request); + if (old) { + GEM_BUG_ON(!i915_request_completed(old)); + __i915_request_submit(old); + i915_request_put(old); + } + + spin_unlock_irq(&ve->base.active.lock); + } + + /* + * Flush the tasklet in case it is still running on another core. + * + * This needs to be done before we remove ourselves from the siblings' + * rbtrees as in the case it is running in parallel, it may reinsert + * the rb_node into a sibling. + */ + tasklet_kill(&ve->base.execlists.tasklet); + + /* Decouple ourselves from the siblings, no more access allowed. */ for (n = 0; n < ve->num_siblings; n++) { struct intel_engine_cs *sibling = ve->siblings[n]; struct rb_node *node = &ve->nodes[sibling->id].rb; - unsigned long flags; if (RB_EMPTY_NODE(node)) continue; - spin_lock_irqsave(&sibling->active.lock, flags); + spin_lock_irq(&sibling->active.lock); /* Detachment is lazily performed in the execlists tasklet */ if (!RB_EMPTY_NODE(node)) rb_erase_cached(node, &sibling->execlists.virtual); - spin_unlock_irqrestore(&sibling->active.lock, flags); + spin_unlock_irq(&sibling->active.lock); } GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet)); + GEM_BUG_ON(!list_empty(virtual_queue(ve))); if (ve->context.state) __execlists_context_fini(&ve->context); intel_context_fini(&ve->context); + intel_breadcrumbs_free(ve->base.breadcrumbs); intel_engine_free_request_pool(&ve->base); kfree(ve->bonds); kfree(ve); } +static void virtual_context_destroy(struct kref *kref) +{ + struct virtual_engine *ve = + container_of(kref, typeof(*ve), context.ref); + + GEM_BUG_ON(!list_empty(&ve->context.signals)); + + /* + * When destroying the virtual engine, we have to be aware that + * it may still be in use from an hardirq/softirq context causing + * the resubmission of a completed request (background completion + * due to preempt-to-busy). Before we can free the engine, we need + * to flush the submission code and tasklets that are still potentially + * accessing the engine. Flushing the tasklets requires process context, + * and since we can guard the resubmit onto the engine with an RCU read + * lock, we can delegate the free of the engine to an RCU worker. + */ + INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy); + queue_rcu_work(system_wq, &ve->rcu); +} + static void virtual_engine_initial_hint(struct virtual_engine *ve) { int swp; @@ -5922,18 +6029,6 @@ int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, return 0; } -struct intel_engine_cs * -intel_virtual_engine_get_sibling(struct intel_engine_cs *engine, - unsigned int sibling) -{ - struct virtual_engine *ve = to_virtual_engine(engine); - - if (sibling >= ve->num_siblings) - return NULL; - - return ve->siblings[sibling]; -} - void intel_execlists_show_requests(struct intel_engine_cs *engine, struct drm_printer *m, void (*show_request)(struct drm_printer *m, diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.h b/drivers/gpu/drm/i915/gt/intel_lrc.h index 91fd8e452d9b..c2d287f25497 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc.h @@ -121,10 +121,6 @@ int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, const struct intel_engine_cs *master, const struct intel_engine_cs *sibling); -struct intel_engine_cs * -intel_virtual_engine_get_sibling(struct intel_engine_cs *engine, - unsigned int sibling); - bool intel_engine_in_execlists_submission_mode(const struct intel_engine_cs *engine); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h index 93cb6c460508..1b51f7b9a5c3 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h @@ -49,4 +49,7 @@ #define GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x1A #define GEN12_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0xD +#define GEN8_EXECLISTS_STATUS_BUF 0x370 +#define GEN11_EXECLISTS_STATUS_BUF2 0x3c0 + #endif /* _INTEL_LRC_REG_H_ */ diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index 39179a3eee98..ab6870242e18 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -59,8 +59,7 @@ struct drm_i915_mocs_table { #define _L3_CACHEABILITY(value) ((value) << 4) /* Helper defines */ -#define GEN9_NUM_MOCS_ENTRIES 62 /* 62 out of 64 - 63 & 64 are reserved. */ -#define GEN11_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ +#define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ /* (e)LLC caching options */ /* @@ -124,14 +123,26 @@ struct drm_i915_mocs_table { LE_1_UC | LE_TC_2_LLC_ELLC, \ L3_1_UC), \ MOCS_ENTRY(I915_MOCS_PTE, \ - LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \ + LE_0_PAGETABLE | LE_TC_0_PAGETABLE | LE_LRUM(3), \ L3_3_WB) static const struct drm_i915_mocs_entry skl_mocs_table[] = { GEN9_MOCS_ENTRIES, MOCS_ENTRY(I915_MOCS_CACHED, LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3), - L3_3_WB) + L3_3_WB), + + /* + * mocs:63 + * - used by the L3 for all of its evictions. + * Thus it is expected to allow LLC cacheability to enable coherent + * flows to be maintained. + * - used to force L3 uncachable cycles. + * Thus it is expected to make the surface L3 uncacheable. + */ + MOCS_ENTRY(63, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC) }; /* NOTE: the LE_TGT_CACHE is not used on Broxton */ @@ -243,8 +254,9 @@ static const struct drm_i915_mocs_entry tgl_mocs_table[] = { * only, __init_mocs_table() take care to program unused index with * this entry. */ - MOCS_ENTRY(1, LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), - L3_3_WB), + MOCS_ENTRY(I915_MOCS_PTE, + LE_0_PAGETABLE | LE_TC_0_PAGETABLE, + L3_1_UC), GEN11_MOCS_ENTRIES, /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ @@ -280,7 +292,7 @@ static const struct drm_i915_mocs_entry icl_mocs_table[] = { L3_1_UC), /* Base - L3 + LeCC:PAT (Deprecated) */ MOCS_ENTRY(I915_MOCS_PTE, - LE_0_PAGETABLE | LE_TC_1_LLC, + LE_0_PAGETABLE | LE_TC_0_PAGETABLE, L3_3_WB), GEN11_MOCS_ENTRIES @@ -348,15 +360,15 @@ static unsigned int get_mocs_settings(const struct drm_i915_private *i915, if (IS_DG1(i915)) { table->size = ARRAY_SIZE(dg1_mocs_table); table->table = dg1_mocs_table; - table->n_entries = GEN11_NUM_MOCS_ENTRIES; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; } else if (INTEL_GEN(i915) >= 12) { table->size = ARRAY_SIZE(tgl_mocs_table); table->table = tgl_mocs_table; - table->n_entries = GEN11_NUM_MOCS_ENTRIES; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; } else if (IS_GEN(i915, 11)) { table->size = ARRAY_SIZE(icl_mocs_table); table->table = icl_mocs_table; - table->n_entries = GEN11_NUM_MOCS_ENTRIES; + table->n_entries = GEN9_NUM_MOCS_ENTRIES; } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) { table->size = ARRAY_SIZE(skl_mocs_table); table->n_entries = GEN9_NUM_MOCS_ENTRIES; diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index ab675d35030d..d7b8e4457fc2 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -56,9 +56,12 @@ static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) static void gen11_rc6_enable(struct intel_rc6 *rc6) { - struct intel_uncore *uncore = rc6_to_uncore(rc6); + struct intel_gt *gt = rc6_to_gt(rc6); + struct intel_uncore *uncore = gt->uncore; struct intel_engine_cs *engine; enum intel_engine_id id; + u32 pg_enable; + int i; /* 2b: Program RC6 thresholds.*/ set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); @@ -102,10 +105,19 @@ static void gen11_rc6_enable(struct intel_rc6 *rc6) GEN6_RC_CTL_RC6_ENABLE | GEN6_RC_CTL_EI_MODE(1); - set(uncore, GEN9_PG_ENABLE, - GEN9_RENDER_PG_ENABLE | - GEN9_MEDIA_PG_ENABLE | - GEN11_MEDIA_SAMPLER_PG_ENABLE); + pg_enable = + GEN9_RENDER_PG_ENABLE | + GEN9_MEDIA_PG_ENABLE | + GEN11_MEDIA_SAMPLER_PG_ENABLE; + + if (INTEL_GEN(gt->i915) >= 12) { + for (i = 0; i < I915_MAX_VCS; i++) + if (HAS_ENGINE(gt, _VCS(i))) + pg_enable |= (VDN_HCP_POWERGATE_ENABLE(i) | + VDN_MFX_POWERGATE_ENABLE(i)); + } + + set(uncore, GEN9_PG_ENABLE, pg_enable); } static void gen9_rc6_enable(struct intel_rc6 *rc6) diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index ac36b67fb46b..3654c955e6be 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -19,6 +19,7 @@ #include "intel_engine_pm.h" #include "intel_gt.h" #include "intel_gt_pm.h" +#include "intel_gt_requests.h" #include "intel_reset.h" #include "uc/intel_guc.h" @@ -1190,14 +1191,14 @@ static void intel_gt_reset_global(struct intel_gt *gt, /* Use a watchdog to ensure that our reset completes */ intel_wedge_on_timeout(&w, gt, 5 * HZ) { - intel_prepare_reset(gt->i915); + intel_display_prepare_reset(gt->i915); /* Flush everyone using a resource about to be clobbered */ synchronize_srcu_expedited(>->reset.backoff_srcu); intel_gt_reset(gt, engine_mask, reason); - intel_finish_reset(gt->i915); + intel_display_finish_reset(gt->i915); } if (!test_bit(I915_WEDGED, >->reset.flags)) @@ -1250,7 +1251,7 @@ void intel_gt_handle_error(struct intel_gt *gt, engine_mask &= gt->info.engine_mask; if (flags & I915_ERROR_CAPTURE) { - i915_capture_error_state(gt->i915); + i915_capture_error_state(gt, engine_mask); intel_gt_clear_error_registers(gt, engine_mask); } @@ -1370,6 +1371,7 @@ void intel_gt_set_wedged_on_fini(struct intel_gt *gt) { intel_gt_set_wedged(gt); set_bit(I915_WEDGED_ON_FINI, >->reset.flags); + intel_gt_retire_requests(gt); /* cleanup any wedged requests */ } void intel_gt_init_reset(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 16b48e72c369..a41b43f445b8 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -444,6 +444,7 @@ static void reset_cancel(struct intel_engine_cs *engine) i915_request_set_error_once(request, -EIO); i915_request_mark_complete(request); } + intel_engine_signal_breadcrumbs(engine); /* Remaining _unready_ requests will be nop'ed when submitted */ diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 466ec671b379..b629eeb14002 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -897,6 +897,10 @@ void intel_rps_park(struct intel_rps *rps) adj = -2; rps->last_adj = adj; rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); + if (rps->cur_freq < rps->efficient_freq) { + rps->cur_freq = rps->efficient_freq; + rps->last_adj = 0; + } GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); } @@ -1973,7 +1977,7 @@ static struct drm_i915_private *mchdev_get(void) rcu_read_lock(); i915 = rcu_dereference(ips_mchdev); - if (!kref_get_unless_zero(&i915->drm.ref)) + if (i915 && !kref_get_unless_zero(&i915->drm.ref)) i915 = NULL; rcu_read_unlock(); diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index fed9503a7c4e..adc9a8ea410a 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -131,8 +131,10 @@ static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) return; } - if (wal->list) + if (wal->list) { memcpy(list, wal->list, sizeof(*wa) * wal->count); + kfree(wal->list); + } wal->list = list; } diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index dfd1cfb8a7ec..2f830017c51d 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -245,18 +245,39 @@ static void mock_reset_rewind(struct intel_engine_cs *engine, bool stalled) GEM_BUG_ON(stalled); } +static void mark_eio(struct i915_request *rq) +{ + if (i915_request_completed(rq)) + return; + + GEM_BUG_ON(i915_request_signaled(rq)); + + i915_request_set_error_once(rq, -EIO); + i915_request_mark_complete(rq); +} + static void mock_reset_cancel(struct intel_engine_cs *engine) { - struct i915_request *request; + struct mock_engine *mock = + container_of(engine, typeof(*mock), base); + struct i915_request *rq; unsigned long flags; + del_timer_sync(&mock->hw_delay); + spin_lock_irqsave(&engine->active.lock, flags); /* Mark all submitted requests as skipped. */ - list_for_each_entry(request, &engine->active.requests, sched.link) { - i915_request_set_error_once(request, -EIO); - i915_request_mark_complete(request); + list_for_each_entry(rq, &engine->active.requests, sched.link) + mark_eio(rq); + intel_engine_signal_breadcrumbs(engine); + + /* Cancel and submit all pending requests. */ + list_for_each_entry(rq, &mock->hw_queue, mock.link) { + mark_eio(rq); + __i915_request_submit(rq); } + INIT_LIST_HEAD(&mock->hw_queue); spin_unlock_irqrestore(&engine->active.lock, flags); } diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index e73854dd2fe0..b88aa35ad75b 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -215,16 +215,17 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine) goto err_pm; for (i = 0; i < ARRAY_SIZE(times); i++) { - /* Manufacture a tick */ do { - while (READ_ONCE(engine->heartbeat.systole)) - flush_delayed_work(&engine->heartbeat.work); + /* Manufacture a tick */ + intel_engine_park_heartbeat(engine); + GEM_BUG_ON(engine->heartbeat.systole); + engine->serial++; /* pretend we are not idle! */ + intel_engine_unpark_heartbeat(engine); - engine->serial++; /* quick, pretend we are not idle! */ flush_delayed_work(&engine->heartbeat.work); if (!delayed_work_pending(&engine->heartbeat.work)) { - pr_err("%s: heartbeat did not start\n", - engine->name); + pr_err("%s: heartbeat %d did not start\n", + engine->name, i); err = -EINVAL; goto err_pm; } diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c index 3540ba9bd459..aa5675ecb5cc 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rps.c +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -219,7 +219,7 @@ int live_rps_clock_interval(void *arg) struct igt_spinner spin; int err = 0; - if (!intel_rps_is_enabled(rps)) + if (!intel_rps_is_enabled(rps) || INTEL_GEN(gt->i915) < 6) return 0; if (igt_spinner_init(&spin, gt)) @@ -1028,7 +1028,7 @@ int live_rps_interrupt(void *arg) * First, let's check whether or not we are receiving interrupts. */ - if (!intel_rps_has_interrupts(rps)) + if (!intel_rps_has_interrupts(rps) || INTEL_GEN(gt->i915) < 6) return 0; intel_gt_pm_get(gt); @@ -1133,7 +1133,7 @@ int live_rps_power(void *arg) * that theory. */ - if (!intel_rps_is_enabled(rps)) + if (!intel_rps_is_enabled(rps) || INTEL_GEN(gt->i915) < 6) return 0; if (!librapl_energy_uJ()) @@ -1237,7 +1237,7 @@ int live_rps_dynamic(void *arg) * moving parts into dynamic reclocking based on load. */ - if (!intel_rps_is_enabled(rps)) + if (!intel_rps_is_enabled(rps) || INTEL_GEN(gt->i915) < 6) return 0; if (igt_spinner_init(&spin, gt)) diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c index 19c2cb166e7c..2edf2b15885f 100644 --- a/drivers/gpu/drm/i915/gt/selftest_timeline.c +++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c @@ -17,8 +17,9 @@ #include "../selftests/i915_random.h" #include "../i915_selftest.h" -#include "../selftests/igt_flush_test.h" -#include "../selftests/mock_gem_device.h" +#include "selftests/igt_flush_test.h" +#include "selftests/lib_sw_fence.h" +#include "selftests/mock_gem_device.h" #include "selftests/mock_timeline.h" static struct page *hwsp_page(struct intel_timeline *tl) @@ -755,6 +756,378 @@ out_free: return err; } +static int emit_read_hwsp(struct i915_request *rq, + u32 seqno, u32 hwsp, + u32 *addr) +{ + const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0)); + u32 *cs; + + cs = intel_ring_begin(rq, 12); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = *addr; + *cs++ = 0; + *cs++ = seqno; + *addr += 4; + + *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = gpr; + *cs++ = hwsp; + *cs++ = 0; + + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; + *cs++ = gpr; + *cs++ = *addr; + *cs++ = 0; + *addr += 4; + + intel_ring_advance(rq, cs); + + return 0; +} + +struct hwsp_watcher { + struct i915_vma *vma; + struct i915_request *rq; + u32 addr; + u32 *map; +}; + +static bool cmp_lt(u32 a, u32 b) +{ + return a < b; +} + +static bool cmp_gte(u32 a, u32 b) +{ + return a >= b; +} + +static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + + obj = i915_gem_object_create_internal(gt->i915, SZ_2M); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + w->map = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(w->map)) { + i915_gem_object_put(obj); + return PTR_ERR(w->map); + } + + vma = i915_gem_object_ggtt_pin_ww(obj, NULL, NULL, 0, 0, 0); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return PTR_ERR(vma); + } + + w->vma = vma; + w->addr = i915_ggtt_offset(vma); + return 0; +} + +static int create_watcher(struct hwsp_watcher *w, + struct intel_engine_cs *engine, + int ringsz) +{ + struct intel_context *ce; + struct intel_timeline *tl; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + ce->ring = __intel_context_ring_size(ringsz); + w->rq = intel_context_create_request(ce); + intel_context_put(ce); + if (IS_ERR(w->rq)) + return PTR_ERR(w->rq); + + w->addr = i915_ggtt_offset(w->vma); + tl = w->rq->context->timeline; + + /* some light mutex juggling required; think co-routines */ + lockdep_unpin_lock(&tl->mutex, w->rq->cookie); + mutex_unlock(&tl->mutex); + + return 0; +} + +static int check_watcher(struct hwsp_watcher *w, const char *name, + bool (*op)(u32 hwsp, u32 seqno)) +{ + struct i915_request *rq = fetch_and_zero(&w->rq); + struct intel_timeline *tl = rq->context->timeline; + u32 offset, end; + int err; + + GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size); + + i915_request_get(rq); + mutex_lock(&tl->mutex); + rq->cookie = lockdep_pin_lock(&tl->mutex); + i915_request_add(rq); + + if (i915_request_wait(rq, 0, HZ) < 0) { + err = -ETIME; + goto out; + } + + err = 0; + offset = 0; + end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map); + while (offset < end) { + if (!op(w->map[offset + 1], w->map[offset])) { + pr_err("Watcher '%s' found HWSP value %x for seqno %x\n", + name, w->map[offset + 1], w->map[offset]); + err = -EINVAL; + } + + offset += 2; + } + +out: + i915_request_put(rq); + return err; +} + +static void cleanup_watcher(struct hwsp_watcher *w) +{ + if (w->rq) { + struct intel_timeline *tl = w->rq->context->timeline; + + mutex_lock(&tl->mutex); + w->rq->cookie = lockdep_pin_lock(&tl->mutex); + + i915_request_add(w->rq); + } + + i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP); +} + +static bool retire_requests(struct intel_timeline *tl) +{ + struct i915_request *rq, *rn; + + mutex_lock(&tl->mutex); + list_for_each_entry_safe(rq, rn, &tl->requests, link) + if (!i915_request_retire(rq)) + break; + mutex_unlock(&tl->mutex); + + return !i915_active_fence_isset(&tl->last_request); +} + +static struct i915_request *wrap_timeline(struct i915_request *rq) +{ + struct intel_context *ce = rq->context; + struct intel_timeline *tl = ce->timeline; + u32 seqno = rq->fence.seqno; + + while (tl->seqno >= seqno) { /* Cause a wrap */ + i915_request_put(rq); + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; + + i915_request_get(rq); + i915_request_add(rq); + } + + i915_request_put(rq); + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return rq; + + i915_request_get(rq); + i915_request_add(rq); + + return rq; +} + +static int live_hwsp_read(void *arg) +{ + struct intel_gt *gt = arg; + struct hwsp_watcher watcher[2] = {}; + struct intel_engine_cs *engine; + struct intel_timeline *tl; + enum intel_engine_id id; + int err = 0; + int i; + + /* + * If we take a reference to the HWSP for reading on the GPU, that + * read may be arbitrarily delayed (either by foreign fence or + * priority saturation) and a wrap can happen within 30 minutes. + * When the GPU read is finally submitted it should be correct, + * even across multiple wraps. + */ + + if (INTEL_GEN(gt->i915) < 8) /* CS convenience [SRM/LRM] */ + return 0; + + tl = intel_timeline_create(gt); + if (IS_ERR(tl)) + return PTR_ERR(tl); + + if (!tl->hwsp_cacheline) + goto out_free; + + for (i = 0; i < ARRAY_SIZE(watcher); i++) { + err = setup_watcher(&watcher[i], gt); + if (err) + goto out; + } + + for_each_engine(engine, gt, id) { + struct intel_context *ce; + unsigned long count = 0; + IGT_TIMEOUT(end_time); + + /* Create a request we can use for remote reading of the HWSP */ + err = create_watcher(&watcher[1], engine, SZ_512K); + if (err) + goto out; + + do { + struct i915_sw_fence *submit; + struct i915_request *rq; + u32 hwsp; + + submit = heap_fence_create(GFP_KERNEL); + if (!submit) { + err = -ENOMEM; + goto out; + } + + err = create_watcher(&watcher[0], engine, SZ_4K); + if (err) + goto out; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto out; + } + + /* Skip to the end, saving 30 minutes of nops */ + tl->seqno = -10u + 2 * (count & 3); + WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); + ce->timeline = intel_timeline_get(tl); + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + intel_context_put(ce); + goto out; + } + + err = i915_sw_fence_await_dma_fence(&rq->submit, + &watcher[0].rq->fence, 0, + GFP_KERNEL); + if (err < 0) { + i915_request_add(rq); + intel_context_put(ce); + goto out; + } + + mutex_lock(&watcher[0].rq->context->timeline->mutex); + err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp); + if (err == 0) + err = emit_read_hwsp(watcher[0].rq, /* before */ + rq->fence.seqno, hwsp, + &watcher[0].addr); + mutex_unlock(&watcher[0].rq->context->timeline->mutex); + if (err) { + i915_request_add(rq); + intel_context_put(ce); + goto out; + } + + mutex_lock(&watcher[1].rq->context->timeline->mutex); + err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp); + if (err == 0) + err = emit_read_hwsp(watcher[1].rq, /* after */ + rq->fence.seqno, hwsp, + &watcher[1].addr); + mutex_unlock(&watcher[1].rq->context->timeline->mutex); + if (err) { + i915_request_add(rq); + intel_context_put(ce); + goto out; + } + + i915_request_get(rq); + i915_request_add(rq); + + rq = wrap_timeline(rq); + intel_context_put(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out; + } + + err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit, + &rq->fence, 0, + GFP_KERNEL); + if (err < 0) { + i915_request_put(rq); + goto out; + } + + err = check_watcher(&watcher[0], "before", cmp_lt); + i915_sw_fence_commit(submit); + heap_fence_put(submit); + if (err) { + i915_request_put(rq); + goto out; + } + count++; + + if (8 * watcher[1].rq->ring->emit > + 3 * watcher[1].rq->ring->size) { + i915_request_put(rq); + break; + } + + /* Flush the timeline before manually wrapping again */ + if (i915_request_wait(rq, + I915_WAIT_INTERRUPTIBLE, + HZ) < 0) { + err = -ETIME; + i915_request_put(rq); + goto out; + } + + retire_requests(tl); + i915_request_put(rq); + } while (!__igt_timeout(end_time, NULL)); + WRITE_ONCE(*(u32 *)tl->hwsp_seqno, 0xdeadbeef); + + pr_info("%s: simulated %lu wraps\n", engine->name, count); + err = check_watcher(&watcher[1], "after", cmp_gte); + if (err) + goto out; + } + +out: + for (i = 0; i < ARRAY_SIZE(watcher); i++) + cleanup_watcher(&watcher[i]); + + if (igt_flush_test(gt->i915)) + err = -EIO; + +out_free: + intel_timeline_put(tl); + return err; +} + static int live_hwsp_rollover_kernel(void *arg) { struct intel_gt *gt = arg; @@ -998,6 +1371,7 @@ int intel_timeline_live_selftests(struct drm_i915_private *i915) SUBTEST(live_hwsp_engine), SUBTEST(live_hwsp_alternate), SUBTEST(live_hwsp_wrap), + SUBTEST(live_hwsp_read), SUBTEST(live_hwsp_rollover_kernel), SUBTEST(live_hwsp_rollover_user), }; diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c index f011ea42487e..5982b62f913d 100644 --- a/drivers/gpu/drm/i915/gt/shmem_utils.c +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -73,7 +73,7 @@ void *shmem_pin_map(struct file *file) mapping_set_unevictable(file->f_mapping); return vaddr; err_page: - while (--i >= 0) + while (i--) put_page(pages[i]); kvfree(pages); return NULL; @@ -103,10 +103,13 @@ static int __shmem_rw(struct file *file, loff_t off, return PTR_ERR(page); vaddr = kmap(page); - if (write) + if (write) { memcpy(vaddr + offset_in_page(off), ptr, this); - else + set_page_dirty(page); + } else { memcpy(ptr, vaddr + offset_in_page(off), this); + } + mark_page_accessed(page); kunmap(page); put_page(page); diff --git a/drivers/gpu/drm/i915/gt/sysfs_engines.c b/drivers/gpu/drm/i915/gt/sysfs_engines.c index 535cc1169e54..967031056202 100644 --- a/drivers/gpu/drm/i915/gt/sysfs_engines.c +++ b/drivers/gpu/drm/i915/gt/sysfs_engines.c @@ -79,14 +79,12 @@ static ssize_t repr_trim(char *buf, ssize_t len) static ssize_t __caps_show(struct intel_engine_cs *engine, - u32 caps, char *buf, bool show_unknown) + unsigned long caps, char *buf, bool show_unknown) { const char * const *repr; int count, n; ssize_t len; - BUILD_BUG_ON(!typecheck(typeof(caps), engine->uabi_capabilities)); - switch (engine->class) { case VIDEO_DECODE_CLASS: repr = vcs_caps; @@ -103,12 +101,10 @@ __caps_show(struct intel_engine_cs *engine, count = 0; break; } - GEM_BUG_ON(count > BITS_PER_TYPE(typeof(caps))); + GEM_BUG_ON(count > BITS_PER_LONG); len = 0; - for_each_set_bit(n, - (unsigned long *)&caps, - show_unknown ? BITS_PER_TYPE(typeof(caps)) : count) { + for_each_set_bit(n, &caps, show_unknown ? BITS_PER_LONG : count) { if (n >= count || !repr[n]) { if (GEM_WARN_ON(show_unknown)) len += snprintf(buf + len, PAGE_SIZE - len, diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index e4aaa5f29796..2a343a977987 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -213,23 +213,6 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) return flags; } -static u32 guc_ctl_ctxinfo_flags(struct intel_guc *guc) -{ - u32 flags = 0; - - if (intel_guc_submission_is_used(guc)) { - u32 ctxnum, base; - - base = intel_guc_ggtt_offset(guc, guc->stage_desc_pool); - ctxnum = GUC_MAX_STAGE_DESCRIPTORS / 16; - - base >>= PAGE_SHIFT; - flags |= (base << GUC_CTL_BASE_ADDR_SHIFT) | - (ctxnum << GUC_CTL_CTXNUM_IN16_SHIFT); - } - return flags; -} - static u32 guc_ctl_log_params_flags(struct intel_guc *guc) { u32 offset = intel_guc_ggtt_offset(guc, guc->log.vma) >> PAGE_SHIFT; @@ -291,7 +274,6 @@ static void guc_init_params(struct intel_guc *guc) BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); - params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc); params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index d44061033f23..5212ff844292 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -10,11 +10,52 @@ /* * The Additional Data Struct (ADS) has pointers for different buffers used by - * the GuC. One single gem object contains the ADS struct itself (guc_ads), the - * scheduling policies (guc_policies), a structure describing a collection of - * register sets (guc_mmio_reg_state) and some extra pages for the GuC to save - * its internal state for sleep. + * the GuC. One single gem object contains the ADS struct itself (guc_ads) and + * all the extra buffers indirectly linked via the ADS struct's entries. + * + * Layout of the ADS blob allocated for the GuC: + * + * +---------------------------------------+ <== base + * | guc_ads | + * +---------------------------------------+ + * | guc_policies | + * +---------------------------------------+ + * | guc_gt_system_info | + * +---------------------------------------+ + * | guc_clients_info | + * +---------------------------------------+ + * | guc_ct_pool_entry[size] | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | private data | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned */ +struct __guc_ads_blob { + struct guc_ads ads; + struct guc_policies policies; + struct guc_gt_system_info system_info; + struct guc_clients_info clients_info; + struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; +} __packed; + +static u32 guc_ads_private_data_size(struct intel_guc *guc) +{ + return PAGE_ALIGN(guc->fw.private_data_size); +} + +static u32 guc_ads_private_data_offset(struct intel_guc *guc) +{ + return PAGE_ALIGN(sizeof(struct __guc_ads_blob)); +} + +static u32 guc_ads_blob_size(struct intel_guc *guc) +{ + return guc_ads_private_data_offset(guc) + + guc_ads_private_data_size(guc); +} static void guc_policy_init(struct guc_policy *policy) { @@ -48,26 +89,37 @@ static void guc_ct_pool_entries_init(struct guc_ct_pool_entry *pool, u32 num) memset(pool, 0, num * sizeof(*pool)); } +static void guc_mapping_table_init(struct intel_gt *gt, + struct guc_gt_system_info *system_info) +{ + unsigned int i, j; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* Table must be set to invalid values for entries not used */ + for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) + for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) + system_info->mapping_table[i][j] = + GUC_MAX_INSTANCES_PER_CLASS; + + for_each_engine(engine, gt, id) { + u8 guc_class = engine->class; + + system_info->mapping_table[guc_class][engine->instance] = + engine->instance; + } +} + /* * The first 80 dwords of the register state context, containing the * execlists and ppgtt registers. */ #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) -/* The ads obj includes the struct itself and buffers passed to GuC */ -struct __guc_ads_blob { - struct guc_ads ads; - struct guc_policies policies; - struct guc_mmio_reg_state reg_state; - struct guc_gt_system_info system_info; - struct guc_clients_info clients_info; - struct guc_ct_pool_entry ct_pool[GUC_CT_POOL_SIZE]; - u8 reg_state_buffer[GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE]; -} __packed; - static void __guc_ads_init(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); + struct drm_i915_private *i915 = gt->i915; struct __guc_ads_blob *blob = guc->ads_blob; const u32 skipped_size = LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE; u32 base; @@ -99,13 +151,25 @@ static void __guc_ads_init(struct intel_guc *guc) } /* System info */ - blob->system_info.slice_enabled = hweight8(gt->info.sseu.slice_mask); - blob->system_info.rcs_enabled = 1; - blob->system_info.bcs_enabled = 1; + blob->system_info.engine_enabled_masks[RENDER_CLASS] = 1; + blob->system_info.engine_enabled_masks[COPY_ENGINE_CLASS] = 1; + blob->system_info.engine_enabled_masks[VIDEO_DECODE_CLASS] = VDBOX_MASK(gt); + blob->system_info.engine_enabled_masks[VIDEO_ENHANCEMENT_CLASS] = VEBOX_MASK(gt); + + blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED] = + hweight8(gt->info.sseu.slice_mask); + blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK] = + gt->info.vdbox_sfc_access; + + if (INTEL_GEN(i915) >= 12 && !IS_DGFX(i915)) { + u32 distdbreg = intel_uncore_read(gt->uncore, + GEN12_DIST_DBS_POPULATED); + blob->system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI] = + ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) & + GEN12_DOORBELLS_PER_SQIDI) + 1; + } - blob->system_info.vdbox_enable_mask = VDBOX_MASK(gt); - blob->system_info.vebox_enable_mask = VEBOX_MASK(gt); - blob->system_info.vdbox_sfc_support_mask = gt->info.vdbox_sfc_access; + guc_mapping_table_init(guc_to_gt(guc), &blob->system_info); base = intel_guc_ggtt_offset(guc, guc->ads_vma); @@ -118,11 +182,12 @@ static void __guc_ads_init(struct intel_guc *guc) /* ADS */ blob->ads.scheduler_policies = base + ptr_offset(blob, policies); - blob->ads.reg_state_buffer = base + ptr_offset(blob, reg_state_buffer); - blob->ads.reg_state_addr = base + ptr_offset(blob, reg_state); blob->ads.gt_system_info = base + ptr_offset(blob, system_info); blob->ads.clients_info = base + ptr_offset(blob, clients_info); + /* Private Data */ + blob->ads.private_data = base + guc_ads_private_data_offset(guc); + i915_gem_object_flush_map(guc->ads_vma->obj); } @@ -135,14 +200,15 @@ static void __guc_ads_init(struct intel_guc *guc) */ int intel_guc_ads_create(struct intel_guc *guc) { - const u32 size = PAGE_ALIGN(sizeof(struct __guc_ads_blob)); + u32 size; int ret; GEM_BUG_ON(guc->ads_vma); + size = guc_ads_blob_size(guc); + ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, (void **)&guc->ads_blob); - if (ret) return ret; @@ -154,6 +220,19 @@ int intel_guc_ads_create(struct intel_guc *guc) void intel_guc_ads_destroy(struct intel_guc *guc) { i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); + guc->ads_blob = NULL; +} + +static void guc_ads_private_data_reset(struct intel_guc *guc) +{ + u32 size; + + size = guc_ads_private_data_size(guc); + if (!size) + return; + + memset((void *)guc->ads_blob + guc_ads_private_data_offset(guc), 0, + size); } /** @@ -168,5 +247,8 @@ void intel_guc_ads_reset(struct intel_guc *guc) { if (!guc->ads_vma) return; + __guc_ads_init(guc); + + guc_ads_private_data_reset(guc); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 11742fca0e9e..fa9e048cc65f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -210,6 +210,7 @@ void intel_guc_ct_fini(struct intel_guc_ct *ct) GEM_BUG_ON(ct->enabled); i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); + memset(ct, 0, sizeof(*ct)); } /** diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c index d4a87f4c9421..f9d0907ea1a5 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c @@ -76,6 +76,7 @@ static inline bool guc_ready(struct intel_uncore *uncore, u32 *status) static int guc_wait_ucode(struct intel_uncore *uncore) { + struct drm_device *drm = &uncore->i915->drm; u32 status; int ret; @@ -90,15 +91,27 @@ static int guc_wait_ucode(struct intel_uncore *uncore) ret = wait_for(guc_ready(uncore, &status), 100); DRM_DEBUG_DRIVER("GuC status %#x\n", status); - if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) { - DRM_ERROR("GuC firmware signature verification failed\n"); - ret = -ENOEXEC; - } - - if ((status & GS_UKERNEL_MASK) == GS_UKERNEL_EXCEPTION) { - DRM_ERROR("GuC firmware exception. EIP: %#x\n", - intel_uncore_read(uncore, SOFT_SCRATCH(13))); - ret = -ENXIO; + if (ret) { + drm_err(drm, "GuC load failed: status = 0x%08X\n", status); + drm_err(drm, "GuC load failed: status: Reset = %d, " + "BootROM = 0x%02X, UKernel = 0x%02X, " + "MIA = 0x%02X, Auth = 0x%02X\n", + REG_FIELD_GET(GS_MIA_IN_RESET, status), + REG_FIELD_GET(GS_BOOTROM_MASK, status), + REG_FIELD_GET(GS_UKERNEL_MASK, status), + REG_FIELD_GET(GS_MIA_MASK, status), + REG_FIELD_GET(GS_AUTH_STATUS_MASK, status)); + + if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) { + drm_err(drm, "GuC firmware signature verification failed\n"); + ret = -ENOEXEC; + } + + if ((status & GS_UKERNEL_MASK) == GS_UKERNEL_EXCEPTION) { + drm_err(drm, "GuC firmware exception. EIP: %#x\n", + intel_uncore_read(uncore, SOFT_SCRATCH(13))); + ret = -ENXIO; + } } return ret; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index a6b733c146c9..79c560d9c0b6 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -26,8 +26,8 @@ #define GUC_VIDEO_ENGINE2 4 #define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) -#define GUC_MAX_ENGINE_CLASSES 5 -#define GUC_MAX_INSTANCES_PER_CLASS 16 +#define GUC_MAX_ENGINE_CLASSES 16 +#define GUC_MAX_INSTANCES_PER_CLASS 32 #define GUC_DOORBELL_INVALID 256 @@ -62,12 +62,7 @@ #define GUC_STAGE_DESC_ATTR_PCH BIT(6) #define GUC_STAGE_DESC_ATTR_TERMINATED BIT(7) -/* New GuC control data */ -#define GUC_CTL_CTXINFO 0 -#define GUC_CTL_CTXNUM_IN16_SHIFT 0 -#define GUC_CTL_BASE_ADDR_SHIFT 12 - -#define GUC_CTL_LOG_PARAMS 1 +#define GUC_CTL_LOG_PARAMS 0 #define GUC_LOG_VALID (1 << 0) #define GUC_LOG_NOTIFY_ON_HALF_FULL (1 << 1) #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) @@ -79,11 +74,11 @@ #define GUC_LOG_ISR_MASK (0x7 << GUC_LOG_ISR_SHIFT) #define GUC_LOG_BUF_ADDR_SHIFT 12 -#define GUC_CTL_WA 2 -#define GUC_CTL_FEATURE 3 +#define GUC_CTL_WA 1 +#define GUC_CTL_FEATURE 2 #define GUC_CTL_DISABLE_SCHEDULER (1 << 14) -#define GUC_CTL_DEBUG 4 +#define GUC_CTL_DEBUG 3 #define GUC_LOG_VERBOSITY_SHIFT 0 #define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) #define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) @@ -97,12 +92,37 @@ #define GUC_LOG_DISABLED (1 << 6) #define GUC_PROFILE_ENABLED (1 << 7) -#define GUC_CTL_ADS 5 +#define GUC_CTL_ADS 4 #define GUC_ADS_ADDR_SHIFT 1 #define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) #define GUC_CTL_MAX_DWORDS (SOFT_SCRATCH_COUNT - 2) /* [1..14] */ +/* Generic GT SysInfo data types */ +#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 +#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 +#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 +#define GUC_GENERIC_GT_SYSINFO_MAX 16 + +/* + * The class goes in bits [0..2] of the GuC ID, the instance in bits [3..6]. + * Bit 7 can be used for operations that apply to all engine classes&instances. + */ +#define GUC_ENGINE_CLASS_SHIFT 0 +#define GUC_ENGINE_CLASS_MASK (0x7 << GUC_ENGINE_CLASS_SHIFT) +#define GUC_ENGINE_INSTANCE_SHIFT 3 +#define GUC_ENGINE_INSTANCE_MASK (0xf << GUC_ENGINE_INSTANCE_SHIFT) +#define GUC_ENGINE_ALL_INSTANCES BIT(7) + +#define MAKE_GUC_ID(class, instance) \ + (((class) << GUC_ENGINE_CLASS_SHIFT) | \ + ((instance) << GUC_ENGINE_INSTANCE_SHIFT)) + +#define GUC_ID_TO_ENGINE_CLASS(guc_id) \ + (((guc_id) & GUC_ENGINE_CLASS_MASK) >> GUC_ENGINE_CLASS_SHIFT) +#define GUC_ID_TO_ENGINE_INSTANCE(guc_id) \ + (((guc_id) & GUC_ENGINE_INSTANCE_MASK) >> GUC_ENGINE_INSTANCE_SHIFT) + /* Work item for submitting workloads into work queue of GuC. */ struct guc_wq_item { u32 header; @@ -336,11 +356,6 @@ struct guc_policies { } __packed; /* GuC MMIO reg state struct */ - - -#define GUC_REGSET_MAX_REGISTERS 64 -#define GUC_S3_SAVE_SPACE_PAGES 10 - struct guc_mmio_reg { u32 offset; u32 value; @@ -348,28 +363,18 @@ struct guc_mmio_reg { #define GUC_REGSET_MASKED (1 << 0) } __packed; -struct guc_mmio_regset { - struct guc_mmio_reg registers[GUC_REGSET_MAX_REGISTERS]; - u32 values_valid; - u32 number_of_registers; -} __packed; - /* GuC register sets */ -struct guc_mmio_reg_state { - struct guc_mmio_regset engine_reg[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; - u32 reserved[98]; +struct guc_mmio_reg_set { + u32 address; + u16 count; + u16 reserved; } __packed; /* HW info */ struct guc_gt_system_info { - u32 slice_enabled; - u32 rcs_enabled; - u32 reserved0; - u32 bcs_enabled; - u32 vdbox_enable_mask; - u32 vdbox_sfc_support_mask; - u32 vebox_enable_mask; - u32 reserved[9]; + u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; + u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; } __packed; /* Clients info */ @@ -390,15 +395,16 @@ struct guc_clients_info { /* GuC Additional Data Struct */ struct guc_ads { - u32 reg_state_addr; - u32 reg_state_buffer; + struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 reserved0; u32 scheduler_policies; u32 gt_system_info; u32 clients_info; u32 control_data; u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; - u32 reserved[16]; + u32 private_data; + u32 reserved[15]; } __packed; /* GuC logging structures */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h index 1949346e714e..b37fc2ffaef2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h @@ -118,6 +118,11 @@ struct guc_doorbell_info { #define GEN8_DRB_VALID (1<<0) #define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) +#define GEN12_DIST_DBS_POPULATED _MMIO(0xd08) +#define GEN12_DOORBELLS_PER_SQIDI_SHIFT 16 +#define GEN12_DOORBELLS_PER_SQIDI (0xff) +#define GEN12_SQIDIS_DOORBELL_EXIST (0xffff) + #define DE_GUCRMR _MMIO(0x44054) #define GUC_BCS_RCS_IER _MMIO(0xC550) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index d6f55f70889d..4e6070e95fe9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -231,13 +231,15 @@ static int guc_enable_communication(struct intel_guc *guc) intel_guc_ct_event_handler(&guc->ct); spin_unlock_irq(&i915->irq_lock); - DRM_INFO("GuC communication enabled\n"); + drm_dbg(&i915->drm, "GuC communication enabled\n"); return 0; } static void guc_disable_communication(struct intel_guc *guc) { + struct drm_i915_private *i915 = guc_to_gt(guc)->i915; + /* * Events generated during or after CT disable are logged by guc in * via mmio. Make sure the register is clear before disabling CT since @@ -257,7 +259,7 @@ static void guc_disable_communication(struct intel_guc *guc) */ guc_get_mmio_msg(guc); - DRM_INFO("GuC communication disabled\n"); + drm_dbg(&i915->drm, "GuC communication disabled\n"); } static void __uc_fetch_firmwares(struct intel_uc *uc) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index 037bcaf3c8b5..180c23e2e25e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -44,24 +44,20 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * List of required GuC and HuC binaries per-platform. * Must be ordered based on platform + revid, from newer to older. * - * TGL 35.2 is interface-compatible with 33.0 for previous Gens. The deltas - * between 33.0 and 35.2 are only related to new additions to support new Gen12 - * features. - * * Note that RKL uses the same firmware as TGL. */ #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ - fw_def(ROCKETLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 5, 0)) \ - fw_def(JASPERLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ - fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ - fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ - fw_def(COMETLAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ - fw_def(COFFEELAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ - fw_def(GEMINILAKE, 0, guc_def(glk, 33, 0, 0), huc_def(glk, 4, 0, 0)) \ - fw_def(KABYLAKE, 0, guc_def(kbl, 33, 0, 0), huc_def(kbl, 4, 0, 0)) \ - fw_def(BROXTON, 0, guc_def(bxt, 33, 0, 0), huc_def(bxt, 2, 0, 0)) \ - fw_def(SKYLAKE, 0, guc_def(skl, 33, 0, 0), huc_def(skl, 2, 0, 0)) + fw_def(ROCKETLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 49, 0, 1), huc_def(tgl, 7, 5, 0)) \ + fw_def(JASPERLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ + fw_def(ELKHARTLAKE, 0, guc_def(ehl, 49, 0, 1), huc_def(ehl, 9, 0, 0)) \ + fw_def(ICELAKE, 0, guc_def(icl, 49, 0, 1), huc_def(icl, 9, 0, 0)) \ + fw_def(COMETLAKE, 5, guc_def(cml, 49, 0, 1), huc_def(cml, 4, 0, 0)) \ + fw_def(COFFEELAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ + fw_def(GEMINILAKE, 0, guc_def(glk, 49, 0, 1), huc_def(glk, 4, 0, 0)) \ + fw_def(KABYLAKE, 0, guc_def(kbl, 49, 0, 1), huc_def(kbl, 4, 0, 0)) \ + fw_def(BROXTON, 0, guc_def(bxt, 49, 0, 1), huc_def(bxt, 2, 0, 0)) \ + fw_def(SKYLAKE, 0, guc_def(skl, 49, 0, 1), huc_def(skl, 2, 0, 0)) #define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ "i915/" \ @@ -372,6 +368,9 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) } } + if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) + uc_fw->private_data_size = css->private_data_size; + obj = i915_gem_object_create_shmem_from_data(i915, fw->data, fw->size); if (IS_ERR(obj)) { err = PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index 23d3a423ac0f..99bb1fe1af66 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -88,6 +88,8 @@ struct intel_uc_fw { u32 rsa_size; u32 ucode_size; + + u32 private_data_size; }; #ifdef CONFIG_DRM_I915_DEBUG_GUC diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h index 029214cdedd5..e41ffc7a7fbc 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h @@ -69,7 +69,11 @@ struct uc_css_header { #define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) #define CSS_SW_VERSION_UC_MINOR (0xFF << 8) #define CSS_SW_VERSION_UC_PATCH (0xFF << 0) - u32 reserved[14]; + u32 reserved0[13]; + union { + u32 private_data_size; /* only applies to GuC */ + u32 reserved1; + }; u32 header_info; } __packed; static_assert(sizeof(struct uc_css_header) == 128); diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index 5b5c71a0b4af..a15f87539657 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -164,7 +164,7 @@ static unsigned char virtual_dp_monitor_edid[GVT_EDID_NUM][EDID_SIZE] = { /* let the virtual display supports DP1.2 */ static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = { - 0x12, 0x014, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + 0x12, 0x014, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; static void emulate_monitor_status_change(struct intel_vgpu *vgpu) @@ -173,22 +173,162 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) int pipe; if (IS_BROXTON(dev_priv)) { + enum transcoder trans; + enum port port; + + /* Clear PIPE, DDI, PHY, HPD before setting new */ vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) &= ~(GEN8_DE_PORT_HOTPLUG(HPD_PORT_A) | GEN8_DE_PORT_HOTPLUG(HPD_PORT_B) | GEN8_DE_PORT_HOTPLUG(HPD_PORT_C)); + for_each_pipe(dev_priv, pipe) { + vgpu_vreg_t(vgpu, PIPECONF(pipe)) &= + ~(PIPECONF_ENABLE | I965_PIPECONF_ACTIVE); + vgpu_vreg_t(vgpu, DSPCNTR(pipe)) &= ~DISPLAY_PLANE_ENABLE; + vgpu_vreg_t(vgpu, SPRCTL(pipe)) &= ~SPRITE_ENABLE; + vgpu_vreg_t(vgpu, CURCNTR(pipe)) &= ~MCURSOR_MODE; + vgpu_vreg_t(vgpu, CURCNTR(pipe)) |= MCURSOR_MODE_DISABLE; + } + + for (trans = TRANSCODER_A; trans <= TRANSCODER_EDP; trans++) { + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(trans)) &= + ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | + TRANS_DDI_PORT_MASK | TRANS_DDI_FUNC_ENABLE); + } + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &= + ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | + TRANS_DDI_PORT_MASK); + + for (port = PORT_A; port <= PORT_C; port++) { + vgpu_vreg_t(vgpu, BXT_PHY_CTL(port)) &= + ~BXT_PHY_LANE_ENABLED; + vgpu_vreg_t(vgpu, BXT_PHY_CTL(port)) |= + (BXT_PHY_CMNLANE_POWERDOWN_ACK | + BXT_PHY_LANE_POWERDOWN_ACK); + + vgpu_vreg_t(vgpu, BXT_PORT_PLL_ENABLE(port)) &= + ~(PORT_PLL_POWER_STATE | PORT_PLL_POWER_ENABLE | + PORT_PLL_REF_SEL | PORT_PLL_LOCK | + PORT_PLL_ENABLE); + + vgpu_vreg_t(vgpu, DDI_BUF_CTL(port)) &= + ~(DDI_INIT_DISPLAY_DETECTED | + DDI_BUF_CTL_ENABLE); + vgpu_vreg_t(vgpu, DDI_BUF_CTL(port)) |= DDI_BUF_IS_IDLE; + } + + vgpu_vreg_t(vgpu, BXT_P_CR_GT_DISP_PWRON) &= ~(BIT(0) | BIT(1)); + vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) &= + ~PHY_POWER_GOOD; + vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) &= + ~PHY_POWER_GOOD; + vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY0)) &= ~BIT(30); + vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY1)) &= ~BIT(30); + + vgpu_vreg_t(vgpu, SFUSE_STRAP) &= ~SFUSE_STRAP_DDIB_DETECTED; + vgpu_vreg_t(vgpu, SFUSE_STRAP) &= ~SFUSE_STRAP_DDIC_DETECTED; + + /* + * Only 1 PIPE enabled in current vGPU display and PIPE_A is + * tied to TRANSCODER_A in HW, so it's safe to assume PIPE_A, + * TRANSCODER_A can be enabled. PORT_x depends on the input of + * setup_virtual_dp_monitor. + */ + vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE; + vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= I965_PIPECONF_ACTIVE; + + /* + * Golden M/N are calculated based on: + * 24 bpp, 4 lanes, 154000 pixel clk (from virtual EDID), + * DP link clk 1620 MHz and non-constant_n. + * TODO: calculate DP link symbol clk and stream clk m/n. + */ + vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) = 63 << TU_SIZE_SHIFT; + vgpu_vreg_t(vgpu, PIPE_DATA_M1(TRANSCODER_A)) |= 0x5b425e; + vgpu_vreg_t(vgpu, PIPE_DATA_N1(TRANSCODER_A)) = 0x800000; + vgpu_vreg_t(vgpu, PIPE_LINK_M1(TRANSCODER_A)) = 0x3cd6e; + vgpu_vreg_t(vgpu, PIPE_LINK_N1(TRANSCODER_A)) = 0x80000; + + /* Enable per-DDI/PORT vreg */ if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { + vgpu_vreg_t(vgpu, BXT_P_CR_GT_DISP_PWRON) |= BIT(1); + vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) |= + PHY_POWER_GOOD; + vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY1)) |= + BIT(30); + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_A)) |= + BXT_PHY_LANE_ENABLED; + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_A)) &= + ~(BXT_PHY_CMNLANE_POWERDOWN_ACK | + BXT_PHY_LANE_POWERDOWN_ACK); + vgpu_vreg_t(vgpu, BXT_PORT_PLL_ENABLE(PORT_A)) |= + (PORT_PLL_POWER_STATE | PORT_PLL_POWER_ENABLE | + PORT_PLL_REF_SEL | PORT_PLL_LOCK | + PORT_PLL_ENABLE); + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_A)) |= + (DDI_BUF_CTL_ENABLE | DDI_INIT_DISPLAY_DETECTED); + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_A)) &= + ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_EDP)) |= + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | + TRANS_DDI_FUNC_ENABLE); vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= GEN8_DE_PORT_HOTPLUG(HPD_PORT_A); } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED; + vgpu_vreg_t(vgpu, BXT_P_CR_GT_DISP_PWRON) |= BIT(0); + vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) |= + PHY_POWER_GOOD; + vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY0)) |= + BIT(30); + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_B)) |= + BXT_PHY_LANE_ENABLED; + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_B)) &= + ~(BXT_PHY_CMNLANE_POWERDOWN_ACK | + BXT_PHY_LANE_POWERDOWN_ACK); + vgpu_vreg_t(vgpu, BXT_PORT_PLL_ENABLE(PORT_B)) |= + (PORT_PLL_POWER_STATE | PORT_PLL_POWER_ENABLE | + PORT_PLL_REF_SEL | PORT_PLL_LOCK | + PORT_PLL_ENABLE); + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) |= + DDI_BUF_CTL_ENABLE; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) &= + ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | + (PORT_B << TRANS_DDI_PORT_SHIFT) | + TRANS_DDI_FUNC_ENABLE); vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= GEN8_DE_PORT_HOTPLUG(HPD_PORT_B); } if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED; + vgpu_vreg_t(vgpu, BXT_P_CR_GT_DISP_PWRON) |= BIT(0); + vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) |= + PHY_POWER_GOOD; + vgpu_vreg_t(vgpu, BXT_PHY_CTL_FAMILY(DPIO_PHY0)) |= + BIT(30); + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_C)) |= + BXT_PHY_LANE_ENABLED; + vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_C)) &= + ~(BXT_PHY_CMNLANE_POWERDOWN_ACK | + BXT_PHY_LANE_POWERDOWN_ACK); + vgpu_vreg_t(vgpu, BXT_PORT_PLL_ENABLE(PORT_C)) |= + (PORT_PLL_POWER_STATE | PORT_PLL_POWER_ENABLE | + PORT_PLL_REF_SEL | PORT_PLL_LOCK | + PORT_PLL_ENABLE); + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) |= + DDI_BUF_CTL_ENABLE; + vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) &= + ~DDI_BUF_IS_IDLE; + vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | + (PORT_B << TRANS_DDI_PORT_SHIFT) | + TRANS_DDI_FUNC_ENABLE); vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= GEN8_DE_PORT_HOTPLUG(HPD_PORT_C); } @@ -520,6 +660,45 @@ void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected) vgpu_vreg_t(vgpu, PCH_PORT_HOTPLUG) |= PORTD_HOTPLUG_STATUS_MASK; intel_vgpu_trigger_virtual_event(vgpu, DP_D_HOTPLUG); + } else if (IS_BROXTON(i915)) { + if (connected) { + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= + GEN8_DE_PORT_HOTPLUG(HPD_PORT_A); + } + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= + SFUSE_STRAP_DDIB_DETECTED; + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= + GEN8_DE_PORT_HOTPLUG(HPD_PORT_B); + } + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) |= + SFUSE_STRAP_DDIC_DETECTED; + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |= + GEN8_DE_PORT_HOTPLUG(HPD_PORT_C); + } + } else { + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) { + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) &= + ~GEN8_DE_PORT_HOTPLUG(HPD_PORT_A); + } + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) &= + ~SFUSE_STRAP_DDIB_DETECTED; + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) &= + ~GEN8_DE_PORT_HOTPLUG(HPD_PORT_B); + } + if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) { + vgpu_vreg_t(vgpu, SFUSE_STRAP) &= + ~SFUSE_STRAP_DDIC_DETECTED; + vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) &= + ~GEN8_DE_PORT_HOTPLUG(HPD_PORT_C); + } + } + vgpu_vreg_t(vgpu, PCH_PORT_HOTPLUG) |= + PORTB_HOTPLUG_STATUS_MASK; + intel_vgpu_trigger_virtual_event(vgpu, DP_B_HOTPLUG); } } diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index a3a4305eda01..897c007ea96a 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -636,9 +636,18 @@ static void ggtt_set_host_entry(struct intel_vgpu_mm *mm, struct intel_gvt_gtt_entry *entry, unsigned long index) { struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; + unsigned long offset = index; GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); + if (vgpu_gmadr_is_aperture(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) { + offset -= (vgpu_aperture_gmadr_base(mm->vgpu) >> PAGE_SHIFT); + mm->ggtt_mm.host_ggtt_aperture[offset] = entry->val64; + } else if (vgpu_gmadr_is_hidden(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) { + offset -= (vgpu_hidden_gmadr_base(mm->vgpu) >> PAGE_SHIFT); + mm->ggtt_mm.host_ggtt_hidden[offset] = entry->val64; + } + pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu); } @@ -1944,6 +1953,21 @@ static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu) return ERR_PTR(-ENOMEM); } + mm->ggtt_mm.host_ggtt_aperture = vzalloc((vgpu_aperture_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64)); + if (!mm->ggtt_mm.host_ggtt_aperture) { + vfree(mm->ggtt_mm.virtual_ggtt); + vgpu_free_mm(mm); + return ERR_PTR(-ENOMEM); + } + + mm->ggtt_mm.host_ggtt_hidden = vzalloc((vgpu_hidden_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64)); + if (!mm->ggtt_mm.host_ggtt_hidden) { + vfree(mm->ggtt_mm.host_ggtt_aperture); + vfree(mm->ggtt_mm.virtual_ggtt); + vgpu_free_mm(mm); + return ERR_PTR(-ENOMEM); + } + return mm; } @@ -1971,6 +1995,8 @@ void _intel_vgpu_mm_release(struct kref *mm_ref) invalidate_ppgtt_mm(mm); } else { vfree(mm->ggtt_mm.virtual_ggtt); + vfree(mm->ggtt_mm.host_ggtt_aperture); + vfree(mm->ggtt_mm.host_ggtt_hidden); } vgpu_free_mm(mm); @@ -2852,3 +2878,41 @@ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu) intel_vgpu_destroy_all_ppgtt_mm(vgpu); intel_vgpu_reset_ggtt(vgpu, true); } + +/** + * intel_gvt_restore_ggtt - restore all vGPU's ggtt entries + * @gvt: intel gvt device + * + * This function is called at driver resume stage to restore + * GGTT entries of every vGPU. + * + */ +void intel_gvt_restore_ggtt(struct intel_gvt *gvt) +{ + struct intel_vgpu *vgpu; + struct intel_vgpu_mm *mm; + int id; + gen8_pte_t pte; + u32 idx, num_low, num_hi, offset; + + /* Restore dirty host ggtt for all vGPUs */ + idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) { + mm = vgpu->gtt.ggtt_mm; + + num_low = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT; + offset = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT; + for (idx = 0; idx < num_low; idx++) { + pte = mm->ggtt_mm.host_ggtt_aperture[idx]; + if (pte & _PAGE_PRESENT) + write_pte64(vgpu->gvt->gt->ggtt, offset + idx, pte); + } + + num_hi = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT; + offset = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT; + for (idx = 0; idx < num_hi; idx++) { + pte = mm->ggtt_mm.host_ggtt_hidden[idx]; + if (pte & _PAGE_PRESENT) + write_pte64(vgpu->gvt->gt->ggtt, offset + idx, pte); + } + } +} diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h index 52d0d88abd86..b0e173f2d990 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.h +++ b/drivers/gpu/drm/i915/gvt/gtt.h @@ -164,6 +164,9 @@ struct intel_vgpu_mm { } ppgtt_mm; struct { void *virtual_ggtt; + /* Save/restore for PM */ + u64 *host_ggtt_aperture; + u64 *host_ggtt_hidden; struct list_head partial_pte_list; } ggtt_mm; }; @@ -280,5 +283,6 @@ int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, void *p_data, unsigned int bytes); void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu); +void intel_gvt_restore_ggtt(struct intel_gvt *gvt); #endif /* _GVT_GTT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index c7c561237883..d1d8ee4a5f16 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -312,7 +312,7 @@ int intel_gvt_init_device(struct drm_i915_private *i915) gvt_dbg_core("init gvt device\n"); - idr_init(&gvt->vgpu_idr); + idr_init_base(&gvt->vgpu_idr, 1); spin_lock_init(&gvt->scheduler.mmio_context_lock); mutex_init(&gvt->lock); mutex_init(&gvt->sched_lock); @@ -406,7 +406,16 @@ out_clean_idr: } int -intel_gvt_register_hypervisor(struct intel_gvt_mpt *m) +intel_gvt_pm_resume(struct intel_gvt *gvt) +{ + intel_gvt_restore_fence(gvt); + intel_gvt_restore_mmio(gvt); + intel_gvt_restore_ggtt(gvt); + return 0; +} + +int +intel_gvt_register_hypervisor(const struct intel_gvt_mpt *m) { int ret; void *gvt; diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 9831361f181e..cf3578e3f4dd 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -56,7 +56,7 @@ struct intel_gvt_host { struct device *dev; bool initialized; int hypervisor_type; - struct intel_gvt_mpt *mpt; + const struct intel_gvt_mpt *mpt; }; extern struct intel_gvt_host intel_gvt_host; @@ -255,7 +255,9 @@ struct intel_gvt_mmio { #define F_CMD_ACCESS (1 << 3) /* This reg has been accessed by a VM */ #define F_ACCESSED (1 << 4) -/* This reg has been accessed through GPU commands */ +/* This reg requires save & restore during host PM suspend/resume */ +#define F_PM_SAVE (1 << 5) +/* This reg could be accessed by unaligned address */ #define F_UNALIGN (1 << 6) /* This reg is in GVT's mmio save-restor list and in hardware * logical context image @@ -685,6 +687,7 @@ void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu); void intel_gvt_debugfs_init(struct intel_gvt *gvt); void intel_gvt_debugfs_clean(struct intel_gvt *gvt); +int intel_gvt_pm_resume(struct intel_gvt *gvt); #include "trace.h" #include "mpt.h" diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index ce93079cf933..aa7e75cb3e6a 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -3120,9 +3120,10 @@ static int init_skl_mmio_info(struct intel_gvt *gvt) MMIO_DFH(TRVATTL3PTRDW(2), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(TRVATTL3PTRDW(3), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); MMIO_DFH(TRVADR, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); - MMIO_DFH(TRTTE, D_SKL_PLUS, F_CMD_ACCESS, - NULL, gen9_trtte_write); - MMIO_DH(_MMIO(0x4dfc), D_SKL_PLUS, NULL, gen9_trtt_chicken_write); + MMIO_DFH(TRTTE, D_SKL_PLUS, F_CMD_ACCESS | F_PM_SAVE, + NULL, gen9_trtte_write); + MMIO_DFH(_MMIO(0x4dfc), D_SKL_PLUS, F_PM_SAVE, + NULL, gen9_trtt_chicken_write); MMIO_D(_MMIO(0x46430), D_SKL_PLUS); @@ -3671,3 +3672,40 @@ default_rw: intel_vgpu_default_mmio_read(vgpu, offset, pdata, bytes) : intel_vgpu_default_mmio_write(vgpu, offset, pdata, bytes); } + +void intel_gvt_restore_fence(struct intel_gvt *gvt) +{ + struct intel_vgpu *vgpu; + int i, id; + + idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) { + mmio_hw_access_pre(gvt->gt); + for (i = 0; i < vgpu_fence_sz(vgpu); i++) + intel_vgpu_write_fence(vgpu, i, vgpu_vreg64(vgpu, fence_num_to_offset(i))); + mmio_hw_access_post(gvt->gt); + } +} + +static inline int mmio_pm_restore_handler(struct intel_gvt *gvt, + u32 offset, void *data) +{ + struct intel_vgpu *vgpu = data; + struct drm_i915_private *dev_priv = gvt->gt->i915; + + if (gvt->mmio.mmio_attribute[offset >> 2] & F_PM_SAVE) + I915_WRITE(_MMIO(offset), vgpu_vreg(vgpu, offset)); + + return 0; +} + +void intel_gvt_restore_mmio(struct intel_gvt *gvt) +{ + struct intel_vgpu *vgpu; + int id; + + idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) { + mmio_hw_access_pre(gvt->gt); + intel_gvt_for_each_tracked_mmio(gvt, mmio_pm_restore_handler, vgpu); + mmio_hw_access_post(gvt->gt); + } +} diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index ad8a9df49f29..60f1a386dd06 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -829,8 +829,10 @@ static int intel_vgpu_open(struct mdev_device *mdev) /* Take a module reference as mdev core doesn't take * a reference for vendor driver. */ - if (!try_module_get(THIS_MODULE)) + if (!try_module_get(THIS_MODULE)) { + ret = -ENODEV; goto undo_group; + } ret = kvmgt_guest_init(mdev); if (ret) @@ -2099,7 +2101,7 @@ static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) return ret; } -static struct intel_gvt_mpt kvmgt_mpt = { +static const struct intel_gvt_mpt kvmgt_mpt = { .type = INTEL_GVT_HYPERVISOR_KVM, .host_init = kvmgt_host_init, .host_exit = kvmgt_host_exit, diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index b6811f6a230d..24210b1eaec5 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -280,6 +280,11 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr) vgpu_vreg_t(vgpu, BXT_PHY_CTL(PORT_C)) |= BXT_PHY_CMNLANE_POWERDOWN_ACK | BXT_PHY_LANE_POWERDOWN_ACK; + vgpu_vreg_t(vgpu, SKL_FUSE_STATUS) |= + SKL_FUSE_DOWNLOAD_STATUS | + SKL_FUSE_PG_DIST_STATUS(SKL_PG0) | + SKL_FUSE_PG_DIST_STATUS(SKL_PG1) | + SKL_FUSE_PG_DIST_STATUS(SKL_PG2); } } else { #define GVT_GEN8_MMIO_RESET_OFFSET (0x44200) diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h index cc4812648bf4..9e862dc73579 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.h +++ b/drivers/gpu/drm/i915/gvt/mmio.h @@ -104,4 +104,8 @@ int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset, int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes); + +void intel_gvt_restore_fence(struct intel_gvt *gvt); +void intel_gvt_restore_mmio(struct intel_gvt *gvt); + #endif diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 9ad224df9c68..6f92cde71971 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -392,7 +392,7 @@ static inline bool intel_gvt_hypervisor_is_valid_gfn( return intel_gvt_host.mpt->is_valid_gfn(vgpu->handle, gfn); } -int intel_gvt_register_hypervisor(struct intel_gvt_mpt *); +int intel_gvt_register_hypervisor(const struct intel_gvt_mpt *); void intel_gvt_unregister_hypervisor(void); #endif /* _GVT_MPT_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index f6d7e33c7099..e49944fde333 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -393,7 +393,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, mutex_init(&vgpu->dmabuf_lock); INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); - idr_init(&vgpu->object_idr); + idr_init_base(&vgpu->object_idr, 1); intel_vgpu_init_cfg_space(vgpu, param->primary); vgpu->d3_entered = false; @@ -439,7 +439,8 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, if (IS_BROADWELL(dev_priv)) ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_B); - else + /* FixMe: Re-enable APL/BXT once vfio_edid enabled */ + else if (!IS_BROXTON(dev_priv)) ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_D); if (ret) goto out_clean_sched_policy; diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index e88970256e8e..93265951fdbb 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1452,43 +1452,42 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, * space. Parsing should be faster in some cases this way. */ batch_end = cmd + batch_length / sizeof(*batch_end); - do { - u32 length; - - if (*cmd == MI_BATCH_BUFFER_END) - break; - - desc = find_cmd(engine, *cmd, desc, &default_desc); - if (!desc) { - DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd); - ret = -EINVAL; - break; - } + while (*cmd != MI_BATCH_BUFFER_END) { + u32 length = 1; + + if (*cmd != MI_NOOP) { /* MI_NOOP == 0 */ + desc = find_cmd(engine, *cmd, desc, &default_desc); + if (!desc) { + DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd); + ret = -EINVAL; + break; + } - if (desc->flags & CMD_DESC_FIXED) - length = desc->length.fixed; - else - length = (*cmd & desc->length.mask) + LENGTH_BIAS; + if (desc->flags & CMD_DESC_FIXED) + length = desc->length.fixed; + else + length = (*cmd & desc->length.mask) + LENGTH_BIAS; - if ((batch_end - cmd) < length) { - DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", - *cmd, - length, - batch_end - cmd); - ret = -EINVAL; - break; - } + if ((batch_end - cmd) < length) { + DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", + *cmd, + length, + batch_end - cmd); + ret = -EINVAL; + break; + } - if (!check_cmd(engine, desc, cmd, length)) { - ret = -EACCES; - break; - } + if (!check_cmd(engine, desc, cmd, length)) { + ret = -EACCES; + break; + } - if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { - ret = check_bbstart(cmd, offset, length, batch_length, - batch_addr, shadow_addr, - jump_whitelist); - break; + if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { + ret = check_bbstart(cmd, offset, length, batch_length, + batch_addr, shadow_addr, + jump_whitelist); + break; + } } if (!IS_ERR_OR_NULL(jump_whitelist)) @@ -1501,7 +1500,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, ret = -EINVAL; break; } - } while (1); + } if (trampoline) { /* diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 200f6b86f864..77e76b665098 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -725,7 +725,7 @@ static int i915_gpu_info_open(struct inode *inode, struct file *file) gpu = NULL; with_intel_runtime_pm(&i915->runtime_pm, wakeref) - gpu = i915_gpu_coredump(i915); + gpu = i915_gpu_coredump(&i915->gt, ALL_ENGINES); if (IS_ERR(gpu)) return PTR_ERR(gpu); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index f2389ba49c69..320856b665a1 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -1271,6 +1271,8 @@ static int i915_drm_resume(struct drm_device *dev) intel_power_domains_enable(dev_priv); + intel_gvt_resume(dev_priv); + enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); return 0; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index d7765b31fbef..15be8debae54 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -508,7 +508,6 @@ struct i915_psr { bool dc3co_enabled; u32 dc3co_exit_delay; struct delayed_work dc3co_work; - bool force_mode_changed; struct drm_dp_vsc_sdp vsc; }; @@ -1560,6 +1559,7 @@ extern const struct i915_rev_steppings kbl_revids[]; (IS_ICELAKE(p) && IS_REVID(p, since, until)) #define EHL_REVID_A0 0x0 +#define EHL_REVID_B0 0x1 #define IS_JSL_EHL_REVID(p, since, until) \ (IS_JSL_EHL(p) && IS_REVID(p, since, until)) @@ -1641,8 +1641,7 @@ tgl_revids_get(struct drm_i915_private *dev_priv) #define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) #define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb) #define HAS_SECURE_BATCHES(dev_priv) (INTEL_GEN(dev_priv) < 6) -#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ - IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) +#define HAS_WT(dev_priv) HAS_EDRAM(dev_priv) #define HWS_NEEDS_PHYSICAL(dev_priv) (INTEL_INFO(dev_priv)->hws_needs_physical) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index bb0c12975f38..58276694c848 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -180,30 +180,6 @@ try_again: } static int -i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) -{ - void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; - char __user *user_data = u64_to_user_ptr(args->data_ptr); - - /* - * We manually control the domain here and pretend that it - * remains coherent i.e. in the GTT domain, like shmem_pwrite. - */ - i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); - - if (copy_from_user(vaddr, user_data, args->size)) - return -EFAULT; - - drm_clflush_virt_range(vaddr, args->size); - intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); - - i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); - return 0; -} - -static int i915_gem_create(struct drm_file *file, struct intel_memory_region *mr, u64 *size_p, @@ -527,6 +503,12 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, trace_i915_gem_object_pread(obj, args->offset, args->size); + ret = -ENODEV; + if (obj->ops->pread) + ret = obj->ops->pread(obj, args); + if (ret != -ENODEV) + goto out; + ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); @@ -866,8 +848,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (ret == -EFAULT || ret == -ENOSPC) { if (i915_gem_object_has_struct_page(obj)) ret = i915_gem_shmem_pwrite(obj, args); - else - ret = i915_gem_phys_pwrite(obj, args, file); } i915_gem_object_unpin_pages(obj); diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index cf6e47adfde6..d8cac4c5881f 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -570,6 +570,7 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, ee->vm_info.pp_dir_base); } } + err_printf(m, " hung: %u\n", ee->hung); err_printf(m, " engine reset count: %u\n", ee->reset_count); for (n = 0; n < ee->num_ports; n++) { @@ -1026,6 +1027,7 @@ i915_vma_coredump_create(const struct intel_gt *gt, dma_addr_t dma; for_each_sgt_daddr(dma, iter, vma->pages) { + mutex_lock(&ggtt->error_mutex); ggtt->vm.insert_page(&ggtt->vm, dma, slot, I915_CACHE_NONE, 0); mb(); @@ -1035,6 +1037,10 @@ i915_vma_coredump_create(const struct intel_gt *gt, (void __force *)s, dst, true); io_mapping_unmap(s); + + mb(); + ggtt->vm.clear_range(&ggtt->vm, slot, PAGE_SIZE); + mutex_unlock(&ggtt->error_mutex); if (ret) break; } @@ -1451,6 +1457,7 @@ capture_engine(struct intel_engine_cs *engine, static void gt_record_engines(struct intel_gt_coredump *gt, + intel_engine_mask_t engine_mask, struct i915_vma_compress *compress) { struct intel_engine_cs *engine; @@ -1466,6 +1473,8 @@ gt_record_engines(struct intel_gt_coredump *gt, if (!ee) continue; + ee->hung = engine->mask & engine_mask; + gt->simulated |= ee->simulated; if (ee->simulated) { kfree(ee); @@ -1505,25 +1514,6 @@ gt_record_uc(struct intel_gt_coredump *gt, return error_uc; } -static void gt_capture_prepare(struct intel_gt_coredump *gt) -{ - struct i915_ggtt *ggtt = gt->_gt->ggtt; - - mutex_lock(&ggtt->error_mutex); -} - -static void gt_capture_finish(struct intel_gt_coredump *gt) -{ - struct i915_ggtt *ggtt = gt->_gt->ggtt; - - if (drm_mm_node_allocated(&ggtt->error_capture)) - ggtt->vm.clear_range(&ggtt->vm, - ggtt->error_capture.start, - PAGE_SIZE); - - mutex_unlock(&ggtt->error_mutex); -} - /* Capture all registers which don't fit into another category. */ static void gt_record_regs(struct intel_gt_coredump *gt) { @@ -1669,24 +1659,25 @@ static u32 generate_ecode(const struct intel_engine_coredump *ee) static const char *error_msg(struct i915_gpu_coredump *error) { struct intel_engine_coredump *first = NULL; + unsigned int hung_classes = 0; struct intel_gt_coredump *gt; - intel_engine_mask_t engines; int len; - engines = 0; for (gt = error->gt; gt; gt = gt->next) { struct intel_engine_coredump *cs; - if (gt->engine && !first) - first = gt->engine; - - for (cs = gt->engine; cs; cs = cs->next) - engines |= cs->engine->mask; + for (cs = gt->engine; cs; cs = cs->next) { + if (cs->hung) { + hung_classes |= BIT(cs->engine->uabi_class); + if (!first) + first = cs; + } + } } len = scnprintf(error->error_msg, sizeof(error->error_msg), "GPU HANG: ecode %d:%x:%08x", - INTEL_GEN(error->i915), engines, + INTEL_GEN(error->i915), hung_classes, generate_ecode(first)); if (first && first->context.pid) { /* Just show the first executing process, more is confusing */ @@ -1782,8 +1773,6 @@ i915_vma_capture_prepare(struct intel_gt_coredump *gt) return NULL; } - gt_capture_prepare(gt); - return compress; } @@ -1793,14 +1782,14 @@ void i915_vma_capture_finish(struct intel_gt_coredump *gt, if (!compress) return; - gt_capture_finish(gt); - compress_fini(compress); kfree(compress); } -struct i915_gpu_coredump *i915_gpu_coredump(struct drm_i915_private *i915) +struct i915_gpu_coredump * +i915_gpu_coredump(struct intel_gt *gt, intel_engine_mask_t engine_mask) { + struct drm_i915_private *i915 = gt->i915; struct i915_gpu_coredump *error; /* Check if GPU capture has been disabled */ @@ -1812,7 +1801,7 @@ struct i915_gpu_coredump *i915_gpu_coredump(struct drm_i915_private *i915) if (!error) return ERR_PTR(-ENOMEM); - error->gt = intel_gt_coredump_alloc(&i915->gt, ALLOW_FAIL); + error->gt = intel_gt_coredump_alloc(gt, ALLOW_FAIL); if (error->gt) { struct i915_vma_compress *compress; @@ -1824,7 +1813,7 @@ struct i915_gpu_coredump *i915_gpu_coredump(struct drm_i915_private *i915) } gt_record_info(error->gt); - gt_record_engines(error->gt, compress); + gt_record_engines(error->gt, engine_mask, compress); if (INTEL_INFO(i915)->has_gt_uc) error->gt->uc = gt_record_uc(error->gt, compress); @@ -1871,20 +1860,23 @@ void i915_error_state_store(struct i915_gpu_coredump *error) /** * i915_capture_error_state - capture an error record for later analysis - * @i915: i915 device + * @gt: intel_gt which originated the hang + * @engine_mask: hung engines + * * * Should be called when an error is detected (either a hang or an error * interrupt) to capture error state from the time of the error. Fills * out a structure which becomes available in debugfs for user level tools * to pick up. */ -void i915_capture_error_state(struct drm_i915_private *i915) +void i915_capture_error_state(struct intel_gt *gt, + intel_engine_mask_t engine_mask) { struct i915_gpu_coredump *error; - error = i915_gpu_coredump(i915); + error = i915_gpu_coredump(gt, engine_mask); if (IS_ERR(error)) { - cmpxchg(&i915->gpu_error.first_error, NULL, error); + cmpxchg(>->i915->gpu_error.first_error, NULL, error); return; } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h index 0220b0992808..16bc42de4b84 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.h +++ b/drivers/gpu/drm/i915/i915_gpu_error.h @@ -59,6 +59,7 @@ struct i915_request_coredump { struct intel_engine_coredump { const struct intel_engine_cs *engine; + bool hung; bool simulated; u32 reset_count; @@ -218,8 +219,10 @@ struct drm_i915_error_state_buf { __printf(2, 3) void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); -struct i915_gpu_coredump *i915_gpu_coredump(struct drm_i915_private *i915); -void i915_capture_error_state(struct drm_i915_private *i915); +struct i915_gpu_coredump *i915_gpu_coredump(struct intel_gt *gt, + intel_engine_mask_t engine_mask); +void i915_capture_error_state(struct intel_gt *gt, + intel_engine_mask_t engine_mask); struct i915_gpu_coredump * i915_gpu_coredump_alloc(struct drm_i915_private *i915, gfp_t gfp); @@ -271,7 +274,8 @@ void i915_disable_error_state(struct drm_i915_private *i915, int err); #else -static inline void i915_capture_error_state(struct drm_i915_private *i915) +static inline void +i915_capture_error_state(struct intel_gt *gt, intel_engine_mask_t engine_mask) { } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index e0eb32bd9607..dc6febc63f1c 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -3058,8 +3058,10 @@ static void gen11_display_irq_reset(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) GEN3_IRQ_RESET(uncore, SDE); - /* Wa_14010685332:icl,jsl,ehl,tgl,rkl */ - if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) { + /* Wa_14010685332:cnp/cmp,tgp,adp */ + if (INTEL_PCH_TYPE(dev_priv) == PCH_CNP || + (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP && + INTEL_PCH_TYPE(dev_priv) < PCH_DG1)) { intel_uncore_rmw(uncore, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS); intel_uncore_rmw(uncore, SOUTH_CHICKEN1, @@ -4204,10 +4206,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) struct drm_device *dev = &dev_priv->drm; int i; - intel_hpd_init_pins(dev_priv); - - intel_hpd_init_work(dev_priv); - INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work); for (i = 0; i < MAX_L3_SLICES; ++i) dev_priv->l3_parity.remap_info[i] = NULL; @@ -4216,6 +4214,13 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11) dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16; + if (!HAS_DISPLAY(dev_priv)) + return; + + intel_hpd_init_pins(dev_priv); + + intel_hpd_init_work(dev_priv); + dev->vblank_disable_immediate = true; /* Most platforms treat the display irq block as an always-on @@ -4237,21 +4242,18 @@ void intel_irq_init(struct drm_i915_private *dev_priv) */ dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv); - if (HAS_GMCH(dev_priv)) { - if (I915_HAS_HOTPLUG(dev_priv)) - dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; - } else { - if (HAS_PCH_DG1(dev_priv)) - dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup; - else if (INTEL_GEN(dev_priv) >= 11) - dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup; - else if (IS_GEN9_LP(dev_priv)) - dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; - else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) - dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; - else - dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; - } + if (HAS_PCH_DG1(dev_priv)) + dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup; + else if (INTEL_GEN(dev_priv) >= 11) + dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup; + else if (IS_GEN9_LP(dev_priv)) + dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup; + else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) + dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup; + else if (HAS_GMCH(dev_priv) && I915_HAS_HOTPLUG(dev_priv)) + dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; + else + dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; } /** diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 1fe390727d80..11fe790b1969 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -1151,9 +1151,13 @@ static int __init i915_init(void) return 0; } + i915_pmu_init(); + err = pci_register_driver(&i915_pci_driver); - if (err) + if (err) { + i915_pmu_exit(); return err; + } i915_perf_sysctl_register(); return 0; @@ -1167,6 +1171,7 @@ static void __exit i915_exit(void) i915_perf_sysctl_unregister(); pci_unregister_driver(&i915_pci_driver); i915_globals_exit(); + i915_pmu_exit(); } module_init(i915_init); diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 3ced82820483..3b12c8ff7182 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -910,8 +910,13 @@ static int gen8_oa_read(struct i915_perf_stream *stream, DRM_I915_PERF_RECORD_OA_REPORT_LOST); if (ret) return ret; - intel_uncore_write(uncore, oastatus_reg, - oastatus & ~GEN8_OASTATUS_REPORT_LOST); + + intel_uncore_rmw(uncore, oastatus_reg, + GEN8_OASTATUS_COUNTER_OVERFLOW | + GEN8_OASTATUS_REPORT_LOST, + IS_GEN_RANGE(uncore->i915, 8, 10) ? + (GEN8_OASTATUS_HEAD_POINTER_WRAP | + GEN8_OASTATUS_TAIL_POINTER_WRAP) : 0); } return gen8_append_oa_reports(stream, buf, count, offset); diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index 69c0fa20eba1..cd786ad12be7 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -30,6 +30,7 @@ #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS) static cpumask_t i915_pmu_cpumask; +static unsigned int i915_pmu_target_cpu = -1; static u8 engine_config_sample(u64 config) { @@ -445,6 +446,8 @@ static void i915_pmu_event_destroy(struct perf_event *event) container_of(event->pmu, typeof(*i915), pmu.base); drm_WARN_ON(&i915->drm, event->parent); + + drm_dev_put(&i915->drm); } static int @@ -510,8 +513,12 @@ static int i915_pmu_event_init(struct perf_event *event) { struct drm_i915_private *i915 = container_of(event->pmu, typeof(*i915), pmu.base); + struct i915_pmu *pmu = &i915->pmu; int ret; + if (pmu->closed) + return -ENODEV; + if (event->attr.type != event->pmu->type) return -ENOENT; @@ -536,8 +543,10 @@ static int i915_pmu_event_init(struct perf_event *event) if (ret) return ret; - if (!event->parent) + if (!event->parent) { + drm_dev_get(&i915->drm); event->destroy = i915_pmu_event_destroy; + } return 0; } @@ -594,9 +603,16 @@ static u64 __i915_pmu_event_read(struct perf_event *event) static void i915_pmu_event_read(struct perf_event *event) { + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); struct hw_perf_event *hwc = &event->hw; + struct i915_pmu *pmu = &i915->pmu; u64 prev, new; + if (pmu->closed) { + event->hw.state = PERF_HES_STOPPED; + return; + } again: prev = local64_read(&hwc->prev_count); new = __i915_pmu_event_read(event); @@ -724,6 +740,13 @@ static void i915_pmu_disable(struct perf_event *event) static void i915_pmu_event_start(struct perf_event *event, int flags) { + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + struct i915_pmu *pmu = &i915->pmu; + + if (pmu->closed) + return; + i915_pmu_enable(event); event->hw.state = 0; } @@ -738,6 +761,13 @@ static void i915_pmu_event_stop(struct perf_event *event, int flags) static int i915_pmu_event_add(struct perf_event *event, int flags) { + struct drm_i915_private *i915 = + container_of(event->pmu, typeof(*i915), pmu.base); + struct i915_pmu *pmu = &i915->pmu; + + if (pmu->closed) + return -ENODEV; + if (flags & PERF_EF_START) i915_pmu_event_start(event, flags); @@ -1020,25 +1050,39 @@ static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) { struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); - unsigned int target; + unsigned int target = i915_pmu_target_cpu; GEM_BUG_ON(!pmu->base.event_init); + /* + * Unregistering an instance generates a CPU offline event which we must + * ignore to avoid incorrectly modifying the shared i915_pmu_cpumask. + */ + if (pmu->closed) + return 0; + if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) { target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); + /* Migrate events if there is a valid target */ if (target < nr_cpu_ids) { cpumask_set_cpu(target, &i915_pmu_cpumask); - perf_pmu_migrate_context(&pmu->base, cpu, target); + i915_pmu_target_cpu = target; } } + if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) { + perf_pmu_migrate_context(&pmu->base, cpu, target); + pmu->cpuhp.cpu = target; + } + return 0; } -static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) +static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; + +void i915_pmu_init(void) { - enum cpuhp_state slot; int ret; ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, @@ -1046,27 +1090,29 @@ static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) i915_pmu_cpu_online, i915_pmu_cpu_offline); if (ret < 0) - return ret; + pr_notice("Failed to setup cpuhp state for i915 PMU! (%d)\n", + ret); + else + cpuhp_slot = ret; +} - slot = ret; - ret = cpuhp_state_add_instance(slot, &pmu->cpuhp.node); - if (ret) { - cpuhp_remove_multi_state(slot); - return ret; - } +void i915_pmu_exit(void) +{ + if (cpuhp_slot != CPUHP_INVALID) + cpuhp_remove_multi_state(cpuhp_slot); +} - pmu->cpuhp.slot = slot; - return 0; +static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) +{ + if (cpuhp_slot == CPUHP_INVALID) + return -EINVAL; + + return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node); } static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu) { - struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); - - drm_WARN_ON(&i915->drm, pmu->cpuhp.slot == CPUHP_INVALID); - drm_WARN_ON(&i915->drm, cpuhp_state_remove_instance(pmu->cpuhp.slot, &pmu->cpuhp.node)); - cpuhp_remove_multi_state(pmu->cpuhp.slot); - pmu->cpuhp.slot = CPUHP_INVALID; + cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node); } static bool is_igp(struct drm_i915_private *i915) @@ -1100,7 +1146,7 @@ void i915_pmu_register(struct drm_i915_private *i915) spin_lock_init(&pmu->lock); hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); pmu->timer.function = i915_sample; - pmu->cpuhp.slot = CPUHP_INVALID; + pmu->cpuhp.cpu = -1; if (!is_igp(i915)) { pmu->name = kasprintf(GFP_KERNEL, @@ -1167,7 +1213,13 @@ void i915_pmu_unregister(struct drm_i915_private *i915) if (!pmu->base.event_init) return; - drm_WARN_ON(&i915->drm, pmu->enable); + /* + * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu + * ensures all currently executing ones will have exited before we + * proceed with unregistration. + */ + pmu->closed = true; + synchronize_rcu(); hrtimer_cancel(&pmu->timer); diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h index 941f0c14037c..a24885ab415c 100644 --- a/drivers/gpu/drm/i915/i915_pmu.h +++ b/drivers/gpu/drm/i915/i915_pmu.h @@ -43,13 +43,17 @@ struct i915_pmu { */ struct { struct hlist_node node; - enum cpuhp_state slot; + unsigned int cpu; } cpuhp; /** * @base: PMU base. */ struct pmu base; /** + * @closed: i915 is unregistering. + */ + bool closed; + /** * @name: Name as registered with perf core. */ const char *name; @@ -122,11 +126,15 @@ struct i915_pmu { }; #ifdef CONFIG_PERF_EVENTS +void i915_pmu_init(void); +void i915_pmu_exit(void); void i915_pmu_register(struct drm_i915_private *i915); void i915_pmu_unregister(struct drm_i915_private *i915); void i915_pmu_gt_parked(struct drm_i915_private *i915); void i915_pmu_gt_unparked(struct drm_i915_private *i915); #else +static inline void i915_pmu_init(void) {} +static inline void i915_pmu_exit(void) {} static inline void i915_pmu_register(struct drm_i915_private *i915) {} static inline void i915_pmu_unregister(struct drm_i915_private *i915) {} static inline void i915_pmu_gt_parked(struct drm_i915_private *i915) {} diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index bb0656875697..5375b219cc3b 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -230,12 +230,14 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) #define _PORT(port, a, b) _PICK_EVEN(port, a, b) #define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) +#define _PHY(phy, a, b) _PICK_EVEN(phy, a, b) #define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) #define _MMIO_PLANE(plane, a, b) _MMIO(_PLANE(plane, a, b)) #define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) #define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) #define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) +#define _MMIO_PHY(phy, a, b) _MMIO(_PHY(phy, a, b)) #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) @@ -677,6 +679,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN7_OASTATUS2_MEM_SELECT_GGTT (1 << 0) /* 0: PPGTT, 1: GGTT */ #define GEN8_OASTATUS _MMIO(0x2b08) +#define GEN8_OASTATUS_TAIL_POINTER_WRAP (1 << 17) +#define GEN8_OASTATUS_HEAD_POINTER_WRAP (1 << 16) #define GEN8_OASTATUS_OVERRUN_STATUS (1 << 3) #define GEN8_OASTATUS_COUNTER_OVERFLOW (1 << 2) #define GEN8_OASTATUS_OABUFFER_OVERFLOW (1 << 1) @@ -8994,10 +8998,6 @@ enum { #define GEN9_PWRGT_MEDIA_STATUS_MASK (1 << 0) #define GEN9_PWRGT_RENDER_STATUS_MASK (1 << 1) -#define POWERGATE_ENABLE _MMIO(0xa210) -#define VDN_HCP_POWERGATE_ENABLE(n) BIT(((n) * 2) + 3) -#define VDN_MFX_POWERGATE_ENABLE(n) BIT(((n) * 2) + 4) - #define GTFIFODBG _MMIO(0x120000) #define GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV (0x1f << 20) #define GT_FIFO_FREE_ENTRIES_CHV (0x7f << 13) @@ -9137,9 +9137,11 @@ enum { #define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xA0C4) #define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xA0C8) #define GEN9_PG_ENABLE _MMIO(0xA210) -#define GEN9_RENDER_PG_ENABLE REG_BIT(0) -#define GEN9_MEDIA_PG_ENABLE REG_BIT(1) -#define GEN11_MEDIA_SAMPLER_PG_ENABLE REG_BIT(2) +#define GEN9_RENDER_PG_ENABLE REG_BIT(0) +#define GEN9_MEDIA_PG_ENABLE REG_BIT(1) +#define GEN11_MEDIA_SAMPLER_PG_ENABLE REG_BIT(2) +#define VDN_HCP_POWERGATE_ENABLE(n) REG_BIT(3 + 2 * (n)) +#define VDN_MFX_POWERGATE_ENABLE(n) REG_BIT(4 + 2 * (n)) #define GEN8_PUSHBUS_CONTROL _MMIO(0xA248) #define GEN8_PUSHBUS_ENABLE _MMIO(0xA250) #define GEN8_PUSHBUS_SHIFT _MMIO(0xA25C) @@ -10300,6 +10302,7 @@ enum skl_power_gate { #define DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port) (3 << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) #define DPCLKA_CFGCR0_DDI_CLK_SEL(pll, port) ((pll) << DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port)) +/* ICL Clocks */ #define ICL_DPCLKA_CFGCR0 _MMIO(0x164280) #define ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) (1 << _PICK(phy, 10, 11, 24)) #define RKL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) REG_BIT((phy) + 10) @@ -10315,6 +10318,27 @@ enum skl_power_gate { #define RKL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy) \ ((pll) << RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) +/* + * DG1 Clocks + * First registers controls the first A and B, while the second register + * controls the phy C and D. The bits on these registers are the + * same, but refer to different phys + */ +#define _DG1_DPCLKA_CFGCR0 0x164280 +#define _DG1_DPCLKA1_CFGCR0 0x16C280 +#define _DG1_DPCLKA_PHY_IDX(phy) ((phy) % 2) +#define _DG1_DPCLKA_PLL_IDX(pll) ((pll) % 2) +#define _DG1_PHY_DPLL_MAP(phy) ((phy) >= PHY_C ? DPLL_ID_DG1_DPLL2 : DPLL_ID_DG1_DPLL0) +#define DG1_DPCLKA_CFGCR0(phy) _MMIO_PHY((phy) / 2, \ + _DG1_DPCLKA_CFGCR0, \ + _DG1_DPCLKA1_CFGCR0) +#define DG1_DPCLKA_CFGCR0_DDI_CLK_OFF(phy) REG_BIT(_DG1_DPCLKA_PHY_IDX(phy) + 10) +#define DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy) (_DG1_DPCLKA_PHY_IDX(phy) * 2) +#define DG1_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy) (_DG1_DPCLKA_PLL_IDX(pll) << DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) +#define DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy) (0x3 << DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) +#define DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_DPLL_MAP(clk_sel, phy) \ + (((clk_sel) >> DG1_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)) + _DG1_PHY_DPLL_MAP(phy)) + /* CNL PLL */ #define DPLL0_ENABLE 0x46010 #define DPLL1_ENABLE 0x46014 diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 16b721080195..620b6fab2c5c 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -176,7 +176,9 @@ struct i915_request { struct intel_context *context; struct intel_ring *ring; struct intel_timeline __rcu *timeline; + struct list_head signal_link; + struct llist_node signal_node; /* * The rcu epoch of when this request was allocated. Used to judiciously diff --git a/drivers/gpu/drm/i915/i915_scatterlist.h b/drivers/gpu/drm/i915/i915_scatterlist.h index 883dd8d09d6b..9cb26a224034 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.h +++ b/drivers/gpu/drm/i915/i915_scatterlist.h @@ -27,13 +27,17 @@ static __always_inline struct sgt_iter { } __sgt_iter(struct scatterlist *sgl, bool dma) { struct sgt_iter s = { .sgp = sgl }; - if (s.sgp) { + if (dma && s.sgp && sg_dma_len(s.sgp) == 0) { + s.sgp = NULL; + } else if (s.sgp) { s.max = s.curr = s.sgp->offset; - s.max += s.sgp->length; - if (dma) + if (dma) { s.dma = sg_dma_address(s.sgp); - else + s.max += sg_dma_len(s.sgp); + } else { s.pfn = page_to_pfn(sg_page(s.sgp)); + s.max += s.sgp->length; + } } return s; @@ -44,6 +48,11 @@ static inline int __sg_page_count(const struct scatterlist *sg) return sg->length >> PAGE_SHIFT; } +static inline int __sg_dma_page_count(const struct scatterlist *sg) +{ + return sg_dma_len(sg) >> PAGE_SHIFT; +} + static inline struct scatterlist *____sg_next(struct scatterlist *sg) { ++sg; diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c index 99fe8aef1c67..4e70c1a9ef2e 100644 --- a/drivers/gpu/drm/i915/intel_gvt.c +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -24,6 +24,7 @@ #include "i915_drv.h" #include "i915_vgpu.h" #include "intel_gvt.h" +#include "gvt/gvt.h" /** * DOC: Intel GVT-g host support @@ -147,3 +148,17 @@ void intel_gvt_driver_remove(struct drm_i915_private *dev_priv) intel_gvt_clean_device(dev_priv); } + +/** + * intel_gvt_resume - GVT resume routine wapper + * + * @dev_priv: drm i915 private * + * + * This function is called at the i915 driver resume stage to restore required + * HW status for GVT so that vGPU can continue running after resumed. + */ +void intel_gvt_resume(struct drm_i915_private *dev_priv) +{ + if (intel_gvt_active(dev_priv)) + intel_gvt_pm_resume(dev_priv->gvt); +} diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h index 502fad8a8652..d7d3fb6186fd 100644 --- a/drivers/gpu/drm/i915/intel_gvt.h +++ b/drivers/gpu/drm/i915/intel_gvt.h @@ -33,6 +33,7 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv); void intel_gvt_clean_device(struct drm_i915_private *dev_priv); int intel_gvt_init_host(void); void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv); +void intel_gvt_resume(struct drm_i915_private *dev_priv); #else static inline int intel_gvt_init(struct drm_i915_private *dev_priv) { @@ -46,6 +47,10 @@ static inline void intel_gvt_driver_remove(struct drm_i915_private *dev_priv) static inline void intel_gvt_sanitize_options(struct drm_i915_private *dev_priv) { } + +static inline void intel_gvt_resume(struct drm_i915_private *dev_priv) +{ +} #endif #endif /* _INTEL_GVT_H_ */ diff --git a/drivers/gpu/drm/i915/intel_memory_region.c b/drivers/gpu/drm/i915/intel_memory_region.c index 180e1078ef7c..b326993a1026 100644 --- a/drivers/gpu/drm/i915/intel_memory_region.c +++ b/drivers/gpu/drm/i915/intel_memory_region.c @@ -114,7 +114,7 @@ __intel_memory_region_get_pages_buddy(struct intel_memory_region *mem, n_pages -= BIT(order); block->private = mem; - list_add(&block->link, blocks); + list_add_tail(&block->link, blocks); if (!n_pages) break; diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index f54375b11964..a20b5051f18c 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -33,6 +33,7 @@ #include <drm/drm_plane_helper.h> #include "display/intel_atomic.h" +#include "display/intel_atomic_plane.h" #include "display/intel_bw.h" #include "display/intel_display_types.h" #include "display/intel_fbc.h" @@ -899,12 +900,12 @@ static void pnv_update_wm(struct intel_crtc *unused_crtc) crtc = single_enabled_crtc(dev_priv); if (crtc) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp = fb->format->cpp[0]; - int clock = adjusted_mode->crtc_clock; + int clock = pipe_mode->crtc_clock; /* Display SR */ wm = intel_calculate_wm(clock, &pnv_display_wm, @@ -1135,8 +1136,8 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; unsigned int latency = dev_priv->wm.pri_latency[level] * 10; unsigned int clock, htotal, cpp, width, wm; @@ -1163,8 +1164,8 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, level != G4X_WM_LEVEL_NORMAL) cpp = max(cpp, 4u); - clock = adjusted_mode->crtc_clock; - htotal = adjusted_mode->crtc_htotal; + clock = pipe_mode->crtc_clock; + htotal = pipe_mode->crtc_htotal; width = drm_rect_width(&plane_state->uapi.dst); @@ -1660,8 +1661,8 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; unsigned int clock, htotal, cpp, width, wm; if (dev_priv->wm.pri_latency[level] == 0) @@ -1671,8 +1672,8 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, return 0; cpp = plane_state->hw.fb->format->cpp[0]; - clock = adjusted_mode->crtc_clock; - htotal = adjusted_mode->crtc_htotal; + clock = pipe_mode->crtc_clock; + htotal = pipe_mode->crtc_htotal; width = crtc_state->pipe_src_w; if (plane->id == PLANE_CURSOR) { @@ -2261,12 +2262,12 @@ static void i965_update_wm(struct intel_crtc *unused_crtc) if (crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 12000; - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; - int clock = adjusted_mode->crtc_clock; - int htotal = adjusted_mode->crtc_htotal; + int clock = pipe_mode->crtc_clock; + int htotal = pipe_mode->crtc_htotal; int hdisplay = crtc->config->pipe_src_w; int cpp = fb->format->cpp[0]; int entries; @@ -2345,8 +2346,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2356,7 +2357,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) else cpp = fb->format->cpp[0]; - planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); enabled = crtc; @@ -2372,8 +2373,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B); crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode = - &crtc->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc->config->hw.pipe_mode; const struct drm_framebuffer *fb = crtc->base.primary->state->fb; int cpp; @@ -2383,7 +2384,7 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) else cpp = fb->format->cpp[0]; - planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); if (enabled == NULL) @@ -2421,12 +2422,12 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc) if (HAS_FW_BLC(dev_priv) && enabled) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; - const struct drm_display_mode *adjusted_mode = - &enabled->config->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &enabled->config->hw.pipe_mode; const struct drm_framebuffer *fb = enabled->base.primary->state->fb; - int clock = adjusted_mode->crtc_clock; - int htotal = adjusted_mode->crtc_htotal; + int clock = pipe_mode->crtc_clock; + int htotal = pipe_mode->crtc_htotal; int hdisplay = enabled->config->pipe_src_w; int cpp; int entries; @@ -2474,7 +2475,7 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) { struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); struct intel_crtc *crtc; - const struct drm_display_mode *adjusted_mode; + const struct drm_display_mode *pipe_mode; u32 fwater_lo; int planea_wm; @@ -2482,8 +2483,8 @@ static void i845_update_wm(struct intel_crtc *unused_crtc) if (crtc == NULL) return; - adjusted_mode = &crtc->config->hw.adjusted_mode; - planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, + pipe_mode = &crtc->config->hw.pipe_mode; + planea_wm = intel_calculate_wm(pipe_mode->crtc_clock, &i845_wm_info, dev_priv->display.get_fifo_size(dev_priv, PLANE_A), 4, pessimal_latency_ns); @@ -2573,7 +2574,7 @@ static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, return method1; method2 = ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); @@ -2601,7 +2602,7 @@ static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value); method2 = ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); return min(method1, method2); @@ -2626,7 +2627,7 @@ static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, cpp = plane_state->hw.fb->format->cpp[0]; return ilk_wm_method2(crtc_state->pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, drm_rect_width(&plane_state->uapi.dst), cpp, mem_value); } @@ -3873,9 +3874,7 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_plane *plane; - const struct intel_plane_state *plane_state; - int level, latency; + enum plane_id plane_id; if (!intel_has_sagv(dev_priv)) return false; @@ -3883,12 +3882,13 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) if (!crtc_state->hw.active) return true; - if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) + if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE) return false; - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { + for_each_plane_id_on_crtc(crtc, plane_id) { const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane->id]; + &crtc_state->wm.skl.optimal.planes[plane_id]; + int level; /* Skip this plane if it's not enabled */ if (!wm->wm[0].plane_en) @@ -3899,19 +3899,12 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) !wm->wm[level].plane_en; --level) { } - latency = dev_priv->wm.skl_latency[level]; - - if (skl_needs_memory_bw_wa(dev_priv) && - plane_state->uapi.fb->modifier == - I915_FORMAT_MOD_X_TILED) - latency += 15; - /* * If any of the planes on this pipe don't enable wm levels that * incur memory latencies higher than sagv_block_time_us we * can't enable SAGV. */ - if (latency < dev_priv->sagv_block_time_us) + if (!wm->wm[level].can_sagv) return false; } @@ -4174,8 +4167,8 @@ skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, */ total_slice_mask = dbuf_slice_mask; for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) { - const struct drm_display_mode *adjusted_mode = - &crtc_state->hw.adjusted_mode; + const struct drm_display_mode *pipe_mode = + &crtc_state->hw.pipe_mode; enum pipe pipe = crtc->pipe; int hdisplay, vdisplay; u32 pipe_dbuf_slice_mask; @@ -4205,7 +4198,7 @@ skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, if (dbuf_slice_mask != pipe_dbuf_slice_mask) continue; - drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay); + drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay); total_width_in_range += hdisplay; @@ -4704,50 +4697,63 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state, } static u64 -skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, - u64 *plane_data_rate, - u64 *uv_plane_data_rate) +skl_get_total_relative_data_rate(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_plane *plane; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; + struct intel_plane *plane; u64 total_data_rate = 0; + enum plane_id plane_id; + int i; /* Calculate and cache data rate for each plane */ - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { - enum plane_id plane_id = plane->id; - u64 rate; + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe != crtc->pipe) + continue; + + plane_id = plane->id; /* packed/y */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); /* uv-plane */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); - uv_plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->uv_plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 1); + } + + for_each_plane_id_on_crtc(crtc, plane_id) { + total_data_rate += crtc_state->plane_data_rate[plane_id]; + total_data_rate += crtc_state->uv_plane_data_rate[plane_id]; } return total_data_rate; } static u64 -icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, - u64 *plane_data_rate) +icl_get_total_relative_data_rate(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_plane *plane; + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; + struct intel_plane *plane; u64 total_data_rate = 0; + enum plane_id plane_id; + int i; /* Calculate and cache data rate for each plane */ - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { - enum plane_id plane_id = plane->id; - u64 rate; + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + if (plane->pipe != crtc->pipe) + continue; + + plane_id = plane->id; if (!plane_state->planar_linked_plane) { - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); } else { enum plane_id y_plane_id; @@ -4762,17 +4768,18 @@ icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, continue; /* Y plane rate is calculated on the slave */ - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0); y_plane_id = plane_state->planar_linked_plane->id; - plane_data_rate[y_plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[y_plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 0); - rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1); - plane_data_rate[plane_id] = rate; - total_data_rate += rate; + crtc_state->plane_data_rate[plane_id] = + skl_plane_relative_data_rate(crtc_state, plane_state, 1); } } + for_each_plane_id_on_crtc(crtc, plane_id) + total_data_rate += crtc_state->plane_data_rate[plane_id]; + return total_data_rate; } @@ -4791,9 +4798,11 @@ skl_plane_wm_level(const struct intel_crtc_state *crtc_state, } static int -skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) +skl_allocate_pipe_ddb(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb; u16 alloc_size, start = 0; @@ -4802,8 +4811,6 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) u64 total_data_rate; enum plane_id plane_id; int num_active; - u64 plane_data_rate[I915_MAX_PLANES] = {}; - u64 uv_plane_data_rate[I915_MAX_PLANES] = {}; u32 blocks; int level; int ret; @@ -4843,13 +4850,10 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) if (INTEL_GEN(dev_priv) >= 11) total_data_rate = - icl_get_total_relative_data_rate(crtc_state, - plane_data_rate); + icl_get_total_relative_data_rate(state, crtc); else total_data_rate = - skl_get_total_relative_data_rate(crtc_state, - plane_data_rate, - uv_plane_data_rate); + skl_get_total_relative_data_rate(state, crtc); ret = skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate, @@ -4930,7 +4934,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) if (total_data_rate == 0) break; - rate = plane_data_rate[plane_id]; + rate = crtc_state->plane_data_rate[plane_id]; extra = min_t(u16, alloc_size, DIV64_U64_ROUND_UP(alloc_size * rate, total_data_rate)); @@ -4941,7 +4945,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) if (total_data_rate == 0) break; - rate = uv_plane_data_rate[plane_id]; + rate = crtc_state->uv_plane_data_rate[plane_id]; extra = min_t(u16, alloc_size, DIV64_U64_ROUND_UP(alloc_size * rate, total_data_rate)); @@ -5093,36 +5097,12 @@ intel_get_linetime_us(const struct intel_crtc_state *crtc_state) if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0)) return u32_to_fixed16(0); - crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal; + crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal; linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); return linetime_us; } -static u32 -skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u64 adjusted_pixel_rate; - uint_fixed_16_16_t downscale_amount; - - /* Shouldn't reach here on disabled planes... */ - if (drm_WARN_ON(&dev_priv->drm, - !intel_wm_plane_visible(crtc_state, plane_state))) - return 0; - - /* - * Adjusted plane pixel rate is just the pipe's adjusted pixel rate - * with additional adjustments for plane-specific scaling. - */ - adjusted_pixel_rate = crtc_state->pixel_rate; - downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state); - - return mul_round_up_u32_fixed16(adjusted_pixel_rate, - downscale_amount); -} - static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, int width, const struct drm_format_info *format, @@ -5235,7 +5215,7 @@ skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, return skl_compute_wm_params(crtc_state, width, fb->format, fb->modifier, plane_state->hw.rotation, - skl_adjusted_plane_pixel_rate(crtc_state, plane_state), + intel_plane_pixel_rate(crtc_state, plane_state), wp, color_plane); } @@ -5282,14 +5262,14 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate, wp->cpp, latency, wp->dbuf_block_size); method2 = skl_wm_method2(wp->plane_pixel_rate, - crtc_state->hw.adjusted_mode.crtc_htotal, + crtc_state->hw.pipe_mode.crtc_htotal, latency, wp->plane_blocks_per_line); if (wp->y_tiled) { selected_result = max_fixed16(method2, wp->y_tile_minimum); } else { - if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal / + if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal / wp->dbuf_block_size < 1) && (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { selected_result = method2; @@ -5373,6 +5353,9 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1; result->plane_en = true; + + if (INTEL_GEN(dev_priv) < 12) + result->can_sagv = latency >= dev_priv->sagv_block_time_us; } static void @@ -5478,7 +5461,7 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; struct skl_wm_params wm_params; int ret; @@ -5501,7 +5484,7 @@ static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, enum plane_id plane_id) { - struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; struct skl_wm_params wm_params; int ret; @@ -5522,10 +5505,13 @@ static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); - const struct drm_framebuffer *fb = plane_state->hw.fb; enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + const struct drm_framebuffer *fb = plane_state->hw.fb; int ret; + memset(wm, 0, sizeof(*wm)); + if (!intel_wm_plane_visible(crtc_state, plane_state)) return 0; @@ -5547,10 +5533,14 @@ static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id; + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; int ret; + memset(wm, 0, sizeof(*wm)); + /* Watermarks calculated in master */ if (plane_state->planar_slave) return 0; @@ -5583,22 +5573,24 @@ static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, return 0; } -static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state) +static int skl_build_pipe_wm(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; - struct intel_plane *plane; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); const struct intel_plane_state *plane_state; - int ret; - - /* - * We'll only calculate watermarks for planes that are actually - * enabled, so make sure all other planes are set as disabled. - */ - memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes)); + struct intel_plane *plane; + int ret, i; - intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, - crtc_state) { + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + /* + * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc + * instead but we don't populate that correctly for NV12 Y + * planes so for now hack this. + */ + if (plane->pipe != crtc->pipe) + continue; if (INTEL_GEN(dev_priv) >= 11) ret = icl_build_plane_wm(crtc_state, plane_state); @@ -5608,6 +5600,8 @@ static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state) return ret; } + crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw; + return 0; } @@ -5794,7 +5788,7 @@ skl_compute_ddb(struct intel_atomic_state *state) for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - ret = skl_allocate_pipe_ddb(new_crtc_state); + ret = skl_allocate_pipe_ddb(state, crtc); if (ret) return ret; @@ -6092,7 +6086,6 @@ skl_compute_wm(struct intel_atomic_state *state) { struct intel_crtc *crtc; struct intel_crtc_state *new_crtc_state; - struct intel_crtc_state *old_crtc_state; int ret, i; ret = skl_ddb_add_affected_pipes(state); @@ -6104,9 +6097,8 @@ skl_compute_wm(struct intel_atomic_state *state) * Note that skl_ddb_add_affected_pipes may have added more CRTC's that * weren't otherwise being modified if pipe allocations had to change. */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - ret = skl_build_pipe_wm(new_crtc_state); + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + ret = skl_build_pipe_wm(state, crtc); if (ret) return ret; } @@ -6124,8 +6116,7 @@ skl_compute_wm(struct intel_atomic_state *state) * based on how much ddb is available. Now we can actually * check if the final watermarks changed. */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { ret = skl_wm_add_affected_planes(state, crtc); if (ret) return ret; @@ -6271,6 +6262,7 @@ void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) crtc_state = to_intel_crtc_state(crtc->base.state); skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); + crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal; } if (dev_priv->active_pipes) { @@ -7111,22 +7103,8 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) 0, CNL_DELAY_PMRSP); } -static void gen12_init_clock_gating(struct drm_i915_private *i915) -{ - unsigned int i; - - /* This is not a WA. Enable VD HCP & MFX_ENC powergate */ - for (i = 0; i < I915_MAX_VCS; i++) - if (HAS_ENGINE(&i915->gt, _VCS(i))) - intel_uncore_rmw(&i915->uncore, POWERGATE_ENABLE, 0, - VDN_HCP_POWERGATE_ENABLE(i) | - VDN_MFX_POWERGATE_ENABLE(i)); -} - static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) { - gen12_init_clock_gating(dev_priv); - /* Wa_1409120013:tgl */ I915_WRITE(ILK_DPFC_CHICKEN, ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL); @@ -7143,8 +7121,6 @@ static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) static void dg1_init_clock_gating(struct drm_i915_private *dev_priv) { - gen12_init_clock_gating(dev_priv); - /* Wa_1409836686:dg1[a0] */ if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0)) I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) | diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index 23a6132c5f4e..412e21604a05 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -211,8 +211,8 @@ static int igt_gem_ww_ctx(void *arg) return PTR_ERR(obj); obj2 = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); + if (IS_ERR(obj2)) { + err = PTR_ERR(obj2); goto put1; } diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index 64bbb8288249..e424a6d1a68c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -2293,8 +2293,10 @@ static int perf_request_latency(void *arg) struct intel_context *ce; ce = intel_context_create(engine); - if (IS_ERR(ce)) + if (IS_ERR(ce)) { + err = PTR_ERR(ce); goto out; + } err = intel_context_pin(ce); if (err) { @@ -2467,8 +2469,10 @@ static int perf_series_engines(void *arg) struct intel_context *ce; ce = intel_context_create(engine); - if (IS_ERR(ce)) + if (IS_ERR(ce)) { + err = PTR_ERR(ce); goto out; + } err = intel_context_pin(ce); if (err) { |