summaryrefslogtreecommitdiffstats
path: root/drivers/misc/habanalabs/gaudi2/gaudi2.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2022-12-16 03:49:24 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2022-12-16 03:49:24 -0800
commitba54ff1fb662215de683777f815b9e96276d55cf (patch)
treed9ad29a17d91fafd76c0b16b41dd30445e50215c /drivers/misc/habanalabs/gaudi2/gaudi2.c
parentdd6f9b17cd7af68b6a5090deedf1f5e84f66f4e6 (diff)
parentf361c96c75184d0272572087c7d9874e0f64b870 (diff)
downloadlinux-ba54ff1fb662215de683777f815b9e96276d55cf.tar.bz2
Merge tag 'char-misc-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc
Pull char/misc driver updates from Greg KH: "Here is the large set of char/misc and other driver subsystem changes for 6.2-rc1. Nothing earth-shattering in here at all, just a lot of new driver development and minor fixes. Highlights include: - fastrpc driver updates - iio new drivers and updates - habanalabs driver updates for new hardware and features - slimbus driver updates - speakup module parameters added to aid in boot time configuration - i2c probe_new conversions for lots of different drivers - other small driver fixes and additions One semi-interesting change in here is the increase of the number of misc dynamic minors available to 1048448 to handle new huge-cpu systems. All of these have been in linux-next for a while with no reported problems" * tag 'char-misc-6.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc: (521 commits) extcon: usbc-tusb320: Convert to i2c's .probe_new() extcon: rt8973: Convert to i2c's .probe_new() extcon: fsa9480: Convert to i2c's .probe_new() extcon: max77843: Replace irqchip mask_invert with unmask_base chardev: fix error handling in cdev_device_add() mcb: mcb-parse: fix error handing in chameleon_parse_gdd() drivers: mcb: fix resource leak in mcb_probe() coresight: etm4x: fix repeated words in comments coresight: cti: Fix null pointer error on CTI init before ETM coresight: trbe: remove cpuhp instance node before remove cpuhp state counter: stm32-lptimer-cnt: fix the check on arr and cmp registers update misc: fastrpc: Add dma_mask to fastrpc_channel_ctx misc: fastrpc: Add mmap request assigning for static PD pool misc: fastrpc: Safekeep mmaps on interrupted invoke misc: fastrpc: Add support for audiopd misc: fastrpc: Rework fastrpc_req_munmap misc: fastrpc: Use fastrpc_map_put in fastrpc_map_create on fail misc: fastrpc: Add fastrpc_remote_heap_alloc misc: fastrpc: Add reserved mem support misc: fastrpc: Rename audio protection domain to root ...
Diffstat (limited to 'drivers/misc/habanalabs/gaudi2/gaudi2.c')
-rw-r--r--drivers/misc/habanalabs/gaudi2/gaudi2.c643
1 files changed, 432 insertions, 211 deletions
diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2.c b/drivers/misc/habanalabs/gaudi2/gaudi2.c
index 65e6cae6100a..e793fb2bdcbe 100644
--- a/drivers/misc/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/misc/habanalabs/gaudi2/gaudi2.c
@@ -128,6 +128,8 @@
#define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
+#define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
+
enum hl_pmmu_fatal_cause {
LATENCY_RD_OUT_FIFO_OVERRUN,
LATENCY_WR_OUT_FIFO_OVERRUN,
@@ -3966,11 +3968,7 @@ static void gaudi2_init_firmware_loader(struct hl_device *hdev)
fw_loader->skip_bmc = false;
fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
fw_loader->dram_bar_id = DRAM_BAR_ID;
-
- if (hdev->asic_type == ASIC_GAUDI2 || hdev->asic_type == ASIC_GAUDI2_SEC)
- fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;
- else /* ASIC_GAUDI2_FPGA */
- fw_loader->cpu_timeout = GAUDI2_FPGA_CPU_TIMEOUT;
+ fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;
/* here we update initial values for few specific dynamic regs (as
* before reading the first descriptor from FW those value has to be
@@ -4471,23 +4469,9 @@ static void gaudi2_init_sm(struct hl_device *hdev)
reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);
WREG32(mmDCORE0_SYNC_MNGR_OBJS_MON_CONFIG_0 + (4 * i), reg_val);
- /* Init CQ0 DB */
- /* Configure the monitor to trigger MSI-X interrupt */
- /* TODO:
- * Remove the if statement when virtual MSI-X doorbell is supported in simulator (SW-93022)
- * and in F/W (SW-93024).
- */
- if (!hdev->pdev || hdev->asic_prop.fw_security_enabled) {
- u64 msix_db_reg = CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF;
-
- WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(msix_db_reg));
- WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(msix_db_reg));
- } else {
- WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0,
- lower_32_bits(gaudi2->virt_msix_db_dma_addr));
- WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0,
- upper_32_bits(gaudi2->virt_msix_db_dma_addr));
- }
+ /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */
+ WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr));
+ WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr));
WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_DATA_0, GAUDI2_IRQ_NUM_COMPLETION);
for (i = 0 ; i < GAUDI2_RESERVED_CQ_NUMBER ; i++) {
@@ -4535,7 +4519,7 @@ static void gaudi2_init_mme_acc(struct hl_device *hdev, u32 reg_base)
static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
bool config_qman_only)
{
- u32 queue_id_base, reg_base, clk_en_addr = 0;
+ u32 queue_id_base, reg_base;
switch (dcore_id) {
case 0:
@@ -4543,23 +4527,18 @@ static void gaudi2_init_dcore_mme(struct hl_device *hdev, int dcore_id,
break;
case 1:
queue_id_base = GAUDI2_QUEUE_ID_DCORE1_MME_0_0;
- clk_en_addr = mmDCORE1_MME_CTRL_LO_QM_SLV_CLK_EN;
break;
case 2:
queue_id_base = GAUDI2_QUEUE_ID_DCORE2_MME_0_0;
break;
case 3:
queue_id_base = GAUDI2_QUEUE_ID_DCORE3_MME_0_0;
- clk_en_addr = mmDCORE3_MME_CTRL_LO_QM_SLV_CLK_EN;
break;
default:
dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);
return;
}
- if (clk_en_addr && !(hdev->fw_components & FW_TYPE_BOOT_CPU))
- WREG32(clk_en_addr, 0x1);
-
if (!config_qman_only) {
reg_base = gaudi2_mme_acc_blocks_bases[dcore_id];
gaudi2_init_mme_acc(hdev, reg_base);
@@ -4660,20 +4639,6 @@ static void gaudi2_init_vdec_brdg_ctrl(struct hl_device *hdev, u64 base_addr, u3
{
u32 sob_id;
- /* TODO:
- * Remove when virtual MSI-X doorbell is supported in simulator (SW-93022) and in F/W
- * (SW-93024).
- */
- if (!hdev->pdev || hdev->asic_prop.fw_security_enabled) {
- u32 interrupt_id = GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 2 * decoder_id;
-
- WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR, mmPCIE_DBI_MSIX_DOORBELL_OFF);
- WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_WDATA, interrupt_id);
- WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_AWADDR, mmPCIE_DBI_MSIX_DOORBELL_OFF);
- WREG32(base_addr + BRDG_CTRL_ABNRM_MSIX_LBW_WDATA, interrupt_id + 1);
- return;
- }
-
/* VCMD normal interrupt */
sob_id = GAUDI2_RESERVED_SOB_DEC_NRM_FIRST + decoder_id;
WREG32(base_addr + BRDG_CTRL_NRM_MSIX_LBW_AWADDR,
@@ -4730,30 +4695,6 @@ static void gaudi2_init_dec(struct hl_device *hdev)
}
}
-static void gaudi2_init_msix_gw_table(struct hl_device *hdev)
-{
- u32 first_reg_offset, last_reg_offset, msix_gw_table_base;
- u8 first_bit, last_bit;
- int i;
-
- msix_gw_table_base = mmPCIE_WRAP_MSIX_GW_TABLE_0;
- first_reg_offset = (GAUDI2_IRQ_NUM_USER_FIRST >> 5) << 2;
- first_bit = GAUDI2_IRQ_NUM_USER_FIRST % 32;
- last_reg_offset = (GAUDI2_IRQ_NUM_USER_LAST >> 5) << 2;
- last_bit = GAUDI2_IRQ_NUM_USER_LAST % 32;
-
- if (first_reg_offset == last_reg_offset) {
- WREG32(msix_gw_table_base + first_reg_offset, GENMASK(last_bit, first_bit));
- return;
- }
-
- WREG32(msix_gw_table_base + first_reg_offset, GENMASK(31, first_bit));
- WREG32(msix_gw_table_base + last_reg_offset, GENMASK(last_bit, 0));
-
- for (i = first_reg_offset + 4; i < last_reg_offset ; i += 4)
- WREG32(msix_gw_table_base + i, 0xFFFFFFFF);
-}
-
static int gaudi2_mmu_update_asid_hop0_addr(struct hl_device *hdev,
u32 stlb_base, u32 asid, u64 phys_addr)
{
@@ -5111,7 +5052,7 @@ static int gaudi2_pci_mmu_init(struct hl_device *hdev)
mmu_base = mmPMMU_HBW_MMU_BASE;
stlb_base = mmPMMU_HBW_STLB_BASE;
- RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
+ RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
(0 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_HOP_SHIFT) |
(5 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_SHIFT) |
(4 << PMMU_HBW_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_SHIFT) |
@@ -5127,7 +5068,7 @@ static int gaudi2_pci_mmu_init(struct hl_device *hdev)
if (PAGE_SIZE == SZ_64K) {
/* Set page sizes to 64K on hop5 and 16M on hop4 + enable 8 bit hops */
- RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET,
+ RMWREG32_SHIFTED(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET,
FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP5_PAGE_SIZE_MASK, 4) |
FIELD_PREP(DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK, 3) |
FIELD_PREP(
@@ -5175,7 +5116,7 @@ static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id,
RMWREG32(mmu_base + MMU_STATIC_MULTI_PAGE_SIZE_OFFSET, 5 /* 64MB */,
MMU_STATIC_MULTI_PAGE_SIZE_HOP4_PAGE_SIZE_MASK);
- RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
+ RMWREG32_SHIFTED(stlb_base + STLB_HOP_CONFIGURATION_OFFSET,
FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_HOP_MASK, 0) |
FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_SMALL_P_MASK, 3) |
FIELD_PREP(DCORE0_HMMU0_STLB_HOP_CONFIGURATION_FIRST_LOOKUP_HOP_LARGE_P_MASK, 3) |
@@ -5267,8 +5208,6 @@ static int gaudi2_hw_init(struct hl_device *hdev)
return rc;
}
- gaudi2_init_msix_gw_table(hdev);
-
gaudi2_init_scrambler_hbm(hdev);
gaudi2_init_kdma(hdev);
@@ -6863,6 +6802,7 @@ static inline bool is_info_event(u32 event)
{
switch (event) {
case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE:
+ case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S ... GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E:
return true;
default:
return false;
@@ -7097,9 +7037,12 @@ static void gaudi2_handle_qman_err_generic(struct hl_device *hdev, const char *q
static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
u64 rtr_mstr_if_base_addr, bool is_write, char *name,
- bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info)
+ bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info,
+ enum gaudi2_engine_id id, u64 *event_mask)
{
u32 razwi_hi, razwi_lo, razwi_xy;
+ u16 eng_id = id;
+ u8 rd_wr_flag;
if (is_write) {
if (read_razwi_regs) {
@@ -7111,6 +7054,7 @@ static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
razwi_lo = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_lo_reg);
razwi_xy = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_id_reg);
}
+ rd_wr_flag = HL_RAZWI_WRITE;
} else {
if (read_razwi_regs) {
razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI);
@@ -7121,8 +7065,12 @@ static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
razwi_lo = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_lo_reg);
razwi_xy = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_id_reg);
}
+ rd_wr_flag = HL_RAZWI_READ;
}
+ hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1,
+ rd_wr_flag | HL_RAZWI_HBW, event_mask);
+
dev_err_ratelimited(hdev->dev,
"%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n",
name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy);
@@ -7130,9 +7078,12 @@ static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev,
static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
u64 rtr_mstr_if_base_addr, bool is_write, char *name,
- bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info)
+ bool read_razwi_regs, struct hl_eq_razwi_info *razwi_info,
+ enum gaudi2_engine_id id, u64 *event_mask)
{
u32 razwi_addr, razwi_xy;
+ u16 eng_id = id;
+ u8 rd_wr_flag;
if (is_write) {
if (read_razwi_regs) {
@@ -7143,9 +7094,7 @@ static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
razwi_xy = le32_to_cpu(razwi_info->lbw.rr_aw_razwi_id_reg);
}
- dev_err_ratelimited(hdev->dev,
- "%s-RAZWI SHARED RR LBW WR error, mstr_if 0x%llx, captured address 0x%x, Initiator coordinates 0x%x\n",
- name, rtr_mstr_if_base_addr, razwi_addr, razwi_xy);
+ rd_wr_flag = HL_RAZWI_WRITE;
} else {
if (read_razwi_regs) {
razwi_addr = RREG32(rtr_mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI);
@@ -7155,9 +7104,57 @@ static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
razwi_xy = le32_to_cpu(razwi_info->lbw.rr_ar_razwi_id_reg);
}
- dev_err_ratelimited(hdev->dev,
- "%s-RAZWI SHARED RR LBW AR error, mstr_if 0x%llx, captured address 0x%x Initiator coordinates 0x%x\n",
- name, rtr_mstr_if_base_addr, razwi_addr, razwi_xy);
+ rd_wr_flag = HL_RAZWI_READ;
+ }
+
+ hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask);
+ dev_err_ratelimited(hdev->dev,
+ "%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%x Initiator coordinates 0x%x\n",
+ name, is_write ? "WR" : "RD", rtr_mstr_if_base_addr, razwi_addr,
+ razwi_xy);
+}
+
+static enum gaudi2_engine_id gaudi2_razwi_calc_engine_id(struct hl_device *hdev,
+ enum razwi_event_sources module, u8 module_idx)
+{
+ switch (module) {
+ case RAZWI_TPC:
+ if (module_idx == (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES))
+ return GAUDI2_DCORE0_ENGINE_ID_TPC_6;
+ return (((module_idx / NUM_OF_TPC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
+ (module_idx % NUM_OF_TPC_PER_DCORE) +
+ (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
+
+ case RAZWI_MME:
+ return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) +
+ (module_idx * ENGINE_ID_DCORE_OFFSET));
+
+ case RAZWI_EDMA:
+ return (((module_idx / NUM_OF_EDMA_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
+ (module_idx % NUM_OF_EDMA_PER_DCORE));
+
+ case RAZWI_PDMA:
+ return (GAUDI2_ENGINE_ID_PDMA_0 + module_idx);
+
+ case RAZWI_NIC:
+ return (GAUDI2_ENGINE_ID_NIC0_0 + (NIC_NUMBER_OF_QM_PER_MACRO * module_idx));
+
+ case RAZWI_DEC:
+ if (module_idx == 8)
+ return GAUDI2_PCIE_ENGINE_ID_DEC_0;
+
+ if (module_idx == 9)
+ return GAUDI2_PCIE_ENGINE_ID_DEC_1;
+ ;
+ return (((module_idx / NUM_OF_DEC_PER_DCORE) * ENGINE_ID_DCORE_OFFSET) +
+ (module_idx % NUM_OF_DEC_PER_DCORE) +
+ (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));
+
+ case RAZWI_ROT:
+ return GAUDI2_ENGINE_ID_ROT_0 + module_idx;
+
+ default:
+ return GAUDI2_ENGINE_ID_SIZE;
}
}
@@ -7167,10 +7164,11 @@ static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev,
*/
static void gaudi2_ack_module_razwi_event_handler(struct hl_device *hdev,
enum razwi_event_sources module, u8 module_idx,
- u8 module_sub_idx, struct hl_eq_razwi_info *razwi_info)
+ u8 module_sub_idx, struct hl_eq_razwi_info *razwi_info,
+ u64 *event_mask)
{
bool via_sft = false, read_razwi_regs = false;
- u32 rtr_id, dcore_id, dcore_rtr_id, sft_id;
+ u32 rtr_id, dcore_id, dcore_rtr_id, sft_id, eng_id;
u64 rtr_mstr_if_base_addr;
u32 hbw_shrd_aw = 0, hbw_shrd_ar = 0;
u32 lbw_shrd_aw = 0, lbw_shrd_ar = 0;
@@ -7304,9 +7302,11 @@ dump_info:
if (!hbw_shrd_aw && !hbw_shrd_ar && !lbw_shrd_aw && !lbw_shrd_ar)
return;
+ eng_id = gaudi2_razwi_calc_engine_id(hdev, module, module_idx);
if (hbw_shrd_aw) {
gaudi2_razwi_rr_hbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, true,
- initiator_name, read_razwi_regs, razwi_info);
+ initiator_name, read_razwi_regs, razwi_info,
+ eng_id, event_mask);
/* Clear event indication */
if (read_razwi_regs)
@@ -7315,7 +7315,8 @@ dump_info:
if (hbw_shrd_ar) {
gaudi2_razwi_rr_hbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, false,
- initiator_name, read_razwi_regs, razwi_info);
+ initiator_name, read_razwi_regs, razwi_info,
+ eng_id, event_mask);
/* Clear event indication */
if (read_razwi_regs)
@@ -7324,7 +7325,8 @@ dump_info:
if (lbw_shrd_aw) {
gaudi2_razwi_rr_lbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, true,
- initiator_name, read_razwi_regs, razwi_info);
+ initiator_name, read_razwi_regs, razwi_info,
+ eng_id, event_mask);
/* Clear event indication */
if (read_razwi_regs)
@@ -7333,7 +7335,8 @@ dump_info:
if (lbw_shrd_ar) {
gaudi2_razwi_rr_lbw_shared_printf_info(hdev, rtr_mstr_if_base_addr, false,
- initiator_name, read_razwi_regs, razwi_info);
+ initiator_name, read_razwi_regs, razwi_info,
+ eng_id, event_mask);
/* Clear event indication */
if (read_razwi_regs)
@@ -7349,38 +7352,42 @@ static void gaudi2_check_if_razwi_happened(struct hl_device *hdev)
/* check all TPCs */
for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) {
if (prop->tpc_enabled_mask & BIT(mod_idx))
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, mod_idx, 0, NULL,
+ NULL);
}
/* check all MMEs */
for (mod_idx = 0 ; mod_idx < (NUM_OF_MME_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
for (sub_mod = MME_WAP0 ; sub_mod < MME_INITIATORS_MAX ; sub_mod++)
gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mod_idx,
- sub_mod, NULL);
+ sub_mod, NULL, NULL);
/* check all EDMAs */
for (mod_idx = 0 ; mod_idx < (NUM_OF_EDMA_PER_DCORE * NUM_OF_DCORES) ; mod_idx++)
if (prop->edma_enabled_mask & BIT(mod_idx))
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_EDMA, mod_idx, 0, NULL,
+ NULL);
/* check all PDMAs */
for (mod_idx = 0 ; mod_idx < NUM_OF_PDMA ; mod_idx++)
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, mod_idx, 0, NULL,
+ NULL);
/* check all NICs */
for (mod_idx = 0 ; mod_idx < NIC_NUMBER_OF_PORTS ; mod_idx++)
if (hdev->nic_ports_mask & BIT(mod_idx))
gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,
- NULL);
+ NULL, NULL);
/* check all DECs */
for (mod_idx = 0 ; mod_idx < NUMBER_OF_DEC ; mod_idx++)
if (prop->decoder_enabled_mask & BIT(mod_idx))
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, mod_idx, 0, NULL,
+ NULL);
/* check all ROTs */
for (mod_idx = 0 ; mod_idx < NUM_OF_ROT ; mod_idx++)
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL, NULL);
}
static const char *gaudi2_get_initiators_name(u32 rtr_id)
@@ -7455,25 +7462,176 @@ static const char *gaudi2_get_initiators_name(u32 rtr_id)
}
}
+static u16 gaudi2_get_razwi_initiators(u32 rtr_id, u16 *engines)
+{
+ switch (rtr_id) {
+ case DCORE0_RTR0:
+ engines[0] = GAUDI2_DCORE0_ENGINE_ID_DEC_0;
+ engines[1] = GAUDI2_DCORE0_ENGINE_ID_DEC_1;
+ engines[2] = GAUDI2_PCIE_ENGINE_ID_DEC_0;
+ engines[3] = GAUDI2_PCIE_ENGINE_ID_DEC_1;
+ engines[4] = GAUDI2_DCORE0_ENGINE_ID_TPC_6;
+ engines[5] = GAUDI2_ENGINE_ID_PDMA_0;
+ engines[6] = GAUDI2_ENGINE_ID_PDMA_1;
+ engines[7] = GAUDI2_ENGINE_ID_PCIE;
+ engines[8] = GAUDI2_DCORE0_ENGINE_ID_EDMA_0;
+ engines[9] = GAUDI2_DCORE1_ENGINE_ID_EDMA_0;
+ engines[10] = GAUDI2_ENGINE_ID_PSOC;
+ return 11;
+
+ case DCORE0_RTR1:
+ engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_0;
+ engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_1;
+ return 2;
+
+ case DCORE0_RTR2:
+ engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_2;
+ engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_3;
+ return 2;
+
+ case DCORE0_RTR3:
+ engines[0] = GAUDI2_DCORE0_ENGINE_ID_TPC_4;
+ engines[1] = GAUDI2_DCORE0_ENGINE_ID_TPC_5;
+ return 2;
+
+ case DCORE0_RTR4:
+ case DCORE0_RTR5:
+ case DCORE0_RTR6:
+ case DCORE0_RTR7:
+ engines[0] = GAUDI2_DCORE0_ENGINE_ID_MME;
+ return 1;
+
+ case DCORE1_RTR0:
+ case DCORE1_RTR1:
+ case DCORE1_RTR2:
+ case DCORE1_RTR3:
+ engines[0] = GAUDI2_DCORE1_ENGINE_ID_MME;
+ return 1;
+
+ case DCORE1_RTR4:
+ engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_4;
+ engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_5;
+ return 2;
+
+ case DCORE1_RTR5:
+ engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_2;
+ engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_3;
+ return 2;
+
+ case DCORE1_RTR6:
+ engines[0] = GAUDI2_DCORE1_ENGINE_ID_TPC_0;
+ engines[1] = GAUDI2_DCORE1_ENGINE_ID_TPC_1;
+ return 2;
+
+ case DCORE1_RTR7:
+ engines[0] = GAUDI2_DCORE1_ENGINE_ID_DEC_0;
+ engines[1] = GAUDI2_DCORE1_ENGINE_ID_DEC_1;
+ engines[2] = GAUDI2_ENGINE_ID_NIC0_0;
+ engines[3] = GAUDI2_ENGINE_ID_NIC1_0;
+ engines[4] = GAUDI2_ENGINE_ID_NIC2_0;
+ engines[5] = GAUDI2_ENGINE_ID_NIC3_0;
+ engines[6] = GAUDI2_ENGINE_ID_NIC4_0;
+ engines[7] = GAUDI2_ENGINE_ID_ARC_FARM;
+ engines[8] = GAUDI2_ENGINE_ID_KDMA;
+ engines[9] = GAUDI2_DCORE0_ENGINE_ID_EDMA_1;
+ engines[10] = GAUDI2_DCORE1_ENGINE_ID_EDMA_1;
+ return 11;
+
+ case DCORE2_RTR0:
+ engines[0] = GAUDI2_DCORE2_ENGINE_ID_DEC_0;
+ engines[1] = GAUDI2_DCORE2_ENGINE_ID_DEC_1;
+ engines[2] = GAUDI2_ENGINE_ID_NIC5_0;
+ engines[3] = GAUDI2_ENGINE_ID_NIC6_0;
+ engines[4] = GAUDI2_ENGINE_ID_NIC7_0;
+ engines[5] = GAUDI2_ENGINE_ID_NIC8_0;
+ engines[6] = GAUDI2_DCORE2_ENGINE_ID_EDMA_0;
+ engines[7] = GAUDI2_DCORE3_ENGINE_ID_EDMA_0;
+ engines[8] = GAUDI2_ENGINE_ID_ROT_0;
+ return 9;
+
+ case DCORE2_RTR1:
+ engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_4;
+ engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_5;
+ return 2;
+
+ case DCORE2_RTR2:
+ engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_2;
+ engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_3;
+ return 2;
+
+ case DCORE2_RTR3:
+ engines[0] = GAUDI2_DCORE2_ENGINE_ID_TPC_0;
+ engines[1] = GAUDI2_DCORE2_ENGINE_ID_TPC_1;
+ return 2;
+
+ case DCORE2_RTR4:
+ case DCORE2_RTR5:
+ case DCORE2_RTR6:
+ case DCORE2_RTR7:
+ engines[0] = GAUDI2_DCORE2_ENGINE_ID_MME;
+ return 1;
+ case DCORE3_RTR0:
+ case DCORE3_RTR1:
+ case DCORE3_RTR2:
+ case DCORE3_RTR3:
+ engines[0] = GAUDI2_DCORE3_ENGINE_ID_MME;
+ return 1;
+ case DCORE3_RTR4:
+ engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_0;
+ engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_1;
+ return 2;
+ case DCORE3_RTR5:
+ engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_2;
+ engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_3;
+ return 2;
+ case DCORE3_RTR6:
+ engines[0] = GAUDI2_DCORE3_ENGINE_ID_TPC_4;
+ engines[1] = GAUDI2_DCORE3_ENGINE_ID_TPC_5;
+ return 2;
+ case DCORE3_RTR7:
+ engines[0] = GAUDI2_DCORE3_ENGINE_ID_DEC_0;
+ engines[1] = GAUDI2_DCORE3_ENGINE_ID_DEC_1;
+ engines[2] = GAUDI2_ENGINE_ID_NIC9_0;
+ engines[3] = GAUDI2_ENGINE_ID_NIC10_0;
+ engines[4] = GAUDI2_ENGINE_ID_NIC11_0;
+ engines[5] = GAUDI2_DCORE2_ENGINE_ID_EDMA_1;
+ engines[6] = GAUDI2_DCORE3_ENGINE_ID_EDMA_1;
+ engines[7] = GAUDI2_ENGINE_ID_ROT_1;
+ engines[8] = GAUDI2_ENGINE_ID_ROT_0;
+ return 9;
+ default:
+ return 0;
+ }
+}
+
static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, u32 rtr_id,
- u64 rtr_ctrl_base_addr, bool is_write)
+ u64 rtr_ctrl_base_addr, bool is_write,
+ u64 *event_mask)
{
+ u16 engines[HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR], num_of_eng;
u32 razwi_hi, razwi_lo;
+ u8 rd_wr_flag;
+
+ num_of_eng = gaudi2_get_razwi_initiators(rtr_id, &engines[0]);
if (is_write) {
razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_HI);
razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_LO);
+ rd_wr_flag = HL_RAZWI_WRITE;
/* Clear set indication */
WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET, 0x1);
} else {
razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_HI);
razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_LO);
+ rd_wr_flag = HL_RAZWI_READ;
/* Clear set indication */
WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET, 0x1);
}
+ hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &engines[0], num_of_eng,
+ rd_wr_flag | HL_RAZWI_HBW, event_mask);
dev_err_ratelimited(hdev->dev,
"RAZWI PSOC unmapped HBW %s error, rtr id %u, address %#llx\n",
is_write ? "WR" : "RD", rtr_id, (u64)razwi_hi << 32 | razwi_lo);
@@ -7483,22 +7641,31 @@ static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, u
}
static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev, u32 rtr_id,
- u64 rtr_ctrl_base_addr, bool is_write)
+ u64 rtr_ctrl_base_addr, bool is_write,
+ u64 *event_mask)
{
+ u16 engines[HL_RAZWI_MAX_NUM_OF_ENGINES_PER_RTR], num_of_eng;
u32 razwi_addr;
+ u8 rd_wr_flag;
+
+ num_of_eng = gaudi2_get_razwi_initiators(rtr_id, &engines[0]);
if (is_write) {
razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_ADDR);
+ rd_wr_flag = HL_RAZWI_WRITE;
/* Clear set indication */
WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET, 0x1);
} else {
razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_ADDR);
+ rd_wr_flag = HL_RAZWI_READ;
/* Clear set indication */
WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET, 0x1);
}
+ hl_handle_razwi(hdev, razwi_addr, &engines[0], num_of_eng, rd_wr_flag | HL_RAZWI_LBW,
+ event_mask);
dev_err_ratelimited(hdev->dev,
"RAZWI PSOC unmapped LBW %s error, rtr id %u, address %#x\n",
is_write ? "WR" : "RD", rtr_id, razwi_addr);
@@ -7508,7 +7675,7 @@ static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev, u
}
/* PSOC RAZWI interrupt occurs only when trying to access a bad address */
-static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev)
+static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev, u64 *event_mask)
{
u32 hbw_aw_set, hbw_ar_set, lbw_aw_set, lbw_ar_set, rtr_id, dcore_id, dcore_rtr_id, xy,
razwi_mask_info, razwi_intr = 0;
@@ -7562,19 +7729,19 @@ static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev)
if (hbw_aw_set)
gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id,
- rtr_ctrl_base_addr, true);
+ rtr_ctrl_base_addr, true, event_mask);
if (hbw_ar_set)
gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id,
- rtr_ctrl_base_addr, false);
+ rtr_ctrl_base_addr, false, event_mask);
if (lbw_aw_set)
gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id,
- rtr_ctrl_base_addr, true);
+ rtr_ctrl_base_addr, true, event_mask);
if (lbw_ar_set)
gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id,
- rtr_ctrl_base_addr, false);
+ rtr_ctrl_base_addr, false, event_mask);
clear:
/* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */
@@ -7600,8 +7767,9 @@ static void _gaudi2_handle_qm_sei_err(struct hl_device *hdev, u64 qman_base)
}
static void gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
- struct hl_eq_razwi_info *razwi_info)
+ struct hl_eq_razwi_info *razwi_info, u64 *event_mask)
{
+ enum razwi_event_sources module;
u64 qman_base;
u8 index;
@@ -7611,9 +7779,11 @@ static void gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
qman_base = mmDCORE0_TPC0_QM_BASE +
(index / NUM_OF_TPC_PER_DCORE) * DCORE_OFFSET +
(index % NUM_OF_TPC_PER_DCORE) * DCORE_TPC_OFFSET;
+ module = RAZWI_TPC;
break;
case GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
qman_base = mmDCORE0_TPC6_QM_BASE;
+ module = RAZWI_TPC;
break;
case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE:
case GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE:
@@ -7623,16 +7793,19 @@ static void gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
qman_base = mmDCORE0_MME_QM_BASE + index * DCORE_OFFSET;
+ module = RAZWI_MME;
break;
case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP;
qman_base = mmPDMA0_QM_BASE + index * PDMA_OFFSET;
+ module = RAZWI_PDMA;
break;
case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
qman_base = mmROT0_QM_BASE + index * ROT_OFFSET;
+ module = RAZWI_ROT;
break;
default:
return;
@@ -7647,7 +7820,7 @@ static void gaudi2_handle_qm_sei_err(struct hl_device *hdev, u16 event_type,
/* check if RAZWI happened */
if (razwi_info)
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_PDMA, 0, 0, razwi_info);
+ gaudi2_ack_module_razwi_event_handler(hdev, module, 0, 0, razwi_info, event_mask);
}
static void gaudi2_handle_qman_err(struct hl_device *hdev, u16 event_type)
@@ -7813,7 +7986,8 @@ static void gaudi2_handle_cpu_sei_err(struct hl_device *hdev)
}
static void gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index,
- struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause)
+ struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
+ u64 *event_mask)
{
u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
int i;
@@ -7825,11 +7999,12 @@ static void gaudi2_handle_rot_err(struct hl_device *hdev, u8 rot_index,
/* check if RAZWI happened */
gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, rot_index, 0,
- &razwi_with_intr_cause->razwi_info);
+ &razwi_with_intr_cause->razwi_info, event_mask);
}
static void gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, char *interrupt_name,
- struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause)
+ struct hl_eq_razwi_with_intr_cause *razwi_with_intr_cause,
+ u64 *event_mask)
{
u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);
int i;
@@ -7841,11 +8016,11 @@ static void gaudi2_tpc_ack_interrupts(struct hl_device *hdev, u8 tpc_index, char
/* check if RAZWI happened */
gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_TPC, tpc_index, 0,
- &razwi_with_intr_cause->razwi_info);
+ &razwi_with_intr_cause->razwi_info, event_mask);
}
static void gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, const char *interrupt_name,
- struct hl_eq_razwi_info *razwi_info)
+ struct hl_eq_razwi_info *razwi_info, u64 *event_mask)
{
u32 sts_addr, sts_val, sts_clr_val = 0;
int i;
@@ -7871,14 +8046,15 @@ static void gaudi2_handle_dec_err(struct hl_device *hdev, u8 dec_index, const ch
}
/* check if RAZWI happened */
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, razwi_info);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_DEC, dec_index, 0, razwi_info,
+ event_mask);
/* Write 1 clear errors */
WREG32(sts_addr, sts_clr_val);
}
static void gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, const char *interrupt_name,
- struct hl_eq_razwi_info *razwi_info)
+ struct hl_eq_razwi_info *razwi_info, u64 *event_mask)
{
u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0;
int i;
@@ -7898,7 +8074,8 @@ static void gaudi2_handle_mme_err(struct hl_device *hdev, u8 mme_index, const ch
/* check if RAZWI happened */
for (i = MME_WRITE ; i < MME_INITIATORS_MAX ; i++)
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, razwi_info);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, i, razwi_info,
+ event_mask);
WREG32(sts_clr_addr, sts_clr_val);
}
@@ -7915,7 +8092,7 @@ static void gaudi2_handle_mme_sbte_err(struct hl_device *hdev, u8 mme_index, u8
}
static void gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index,
- struct hl_eq_razwi_info *razwi_info)
+ struct hl_eq_razwi_info *razwi_info, u64 *event_mask)
{
u32 sts_addr, sts_val, sts_clr_addr, sts_clr_val = 0;
int i;
@@ -7935,8 +8112,10 @@ static void gaudi2_handle_mme_wap_err(struct hl_device *hdev, u8 mme_index,
}
/* check if RAZWI happened on WAP0/1 */
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, razwi_info);
- gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, razwi_info);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP0, razwi_info,
+ event_mask);
+ gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_MME, mme_index, MME_WAP1, razwi_info,
+ event_mask);
WREG32(sts_clr_addr, sts_clr_val);
}
@@ -7966,40 +8145,41 @@ static void gaudi2_handle_dma_core_event(struct hl_device *hdev, u64 intr_cause_
gaudi2_dma_core_interrupts_cause[i]);
}
-static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev)
+static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev, u64 *event_mask)
{
u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr;
razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED;
if (RREG32(razwi_happened_addr)) {
gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", true,
- NULL);
+ NULL, GAUDI2_ENGINE_ID_PCIE, event_mask);
WREG32(razwi_happened_addr, 0x1);
}
razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED;
if (RREG32(razwi_happened_addr)) {
gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", true,
- NULL);
+ NULL, GAUDI2_ENGINE_ID_PCIE, event_mask);
WREG32(razwi_happened_addr, 0x1);
}
razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED;
if (RREG32(razwi_happened_addr)) {
gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", true,
- NULL);
+ NULL, GAUDI2_ENGINE_ID_PCIE, event_mask);
WREG32(razwi_happened_addr, 0x1);
}
razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED;
if (RREG32(razwi_happened_addr)) {
gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", true,
- NULL);
+ NULL, GAUDI2_ENGINE_ID_PCIE, event_mask);
WREG32(razwi_happened_addr, 0x1);
}
}
-static void gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u64 intr_cause_data)
+static void gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u64 intr_cause_data,
+ u64 *event_mask)
{
int i;
@@ -8014,7 +8194,7 @@ static void gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u64 intr_cau
case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK:
break;
case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK:
- gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev);
+ gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev, event_mask);
break;
}
}
@@ -8047,7 +8227,8 @@ static void gaudi2_handle_hif_fatal(struct hl_device *hdev, u16 event_type, u64
}
}
-static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu)
+static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool is_pmmu,
+ u64 *event_mask)
{
u32 valid, val;
u64 addr;
@@ -8064,6 +8245,7 @@ static void gaudi2_handle_page_error(struct hl_device *hdev, u64 mmu_base, bool
dev_err_ratelimited(hdev->dev, "%s page fault on va 0x%llx\n",
is_pmmu ? "PMMU" : "HMMU", addr);
+ hl_handle_page_fault(hdev, addr, 0, is_pmmu, event_mask);
WREG32(mmu_base + MMU_OFFSET(mmDCORE0_HMMU0_MMU_PAGE_ERROR_CAPTURE), 0);
}
@@ -8089,7 +8271,7 @@ static void gaudi2_handle_access_error(struct hl_device *hdev, u64 mmu_base, boo
}
static void gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, const char *mmu_name,
- u64 mmu_base, bool is_pmmu)
+ u64 mmu_base, bool is_pmmu, u64 *event_mask)
{
u32 spi_sei_cause, interrupt_clr = 0x0;
int i;
@@ -8102,7 +8284,7 @@ static void gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, const char
mmu_name, gaudi2_mmu_spi_sei[i].cause);
if (i == 0)
- gaudi2_handle_page_error(hdev, mmu_base, is_pmmu);
+ gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, event_mask);
else if (i == 1)
gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
@@ -8118,11 +8300,10 @@ static void gaudi2_handle_mmu_spi_sei_generic(struct hl_device *hdev, const char
WREG32(mmu_base + MMU_INTERRUPT_CLR_OFFSET, interrupt_clr);
}
-static bool gaudi2_handle_sm_err(struct hl_device *hdev, u8 sm_index)
+static void gaudi2_handle_sm_err(struct hl_device *hdev, u8 sm_index)
{
u32 sei_cause_addr, sei_cause_val, sei_cause_cause, sei_cause_log;
u32 cq_intr_addr, cq_intr_val, cq_intr_queue_index;
- bool reset = true;
int i;
sei_cause_addr = mmDCORE0_SYNC_MNGR_GLBL_SM_SEI_CAUSE + DCORE_OFFSET * sm_index;
@@ -8147,10 +8328,6 @@ static bool gaudi2_handle_sm_err(struct hl_device *hdev, u8 sm_index)
gaudi2_sm_sei_cause[i].cause_name,
gaudi2_sm_sei_cause[i].log_name,
sei_cause_log & gaudi2_sm_sei_cause[i].log_mask);
-
- /* Due to a potential H/W issue, do not reset upon BRESP errors */
- if (i == 2)
- reset = false;
break;
}
@@ -8170,11 +8347,9 @@ static bool gaudi2_handle_sm_err(struct hl_device *hdev, u8 sm_index)
/* Clear CQ_INTR */
WREG32(cq_intr_addr, 0);
}
-
- return reset;
}
-static void gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type)
+static void gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)
{
bool is_pmmu = false;
char desc[32];
@@ -8232,7 +8407,7 @@ static void gaudi2_handle_mmu_spi_sei_err(struct hl_device *hdev, u16 event_type
return;
}
- gaudi2_handle_mmu_spi_sei_generic(hdev, desc, mmu_base, is_pmmu);
+ gaudi2_handle_mmu_spi_sei_generic(hdev, desc, mmu_base, is_pmmu, event_mask);
}
@@ -8476,8 +8651,8 @@ static void gaudi2_print_out_of_sync_info(struct hl_device *hdev,
{
struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
- dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
- sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
+ dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",
+ le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));
}
static void gaudi2_handle_pcie_p2p_msix(struct hl_device *hdev)
@@ -8543,8 +8718,8 @@ static void gaudi2_print_cpu_pkt_failure_info(struct hl_device *hdev,
struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];
dev_warn(hdev->dev,
- "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%u\n",
- sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
+ "FW reported sanity check failure, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",
+ le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));
}
static void hl_arc_event_handle(struct hl_device *hdev,
@@ -8573,9 +8748,9 @@ static void hl_arc_event_handle(struct hl_device *hdev,
static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
{
- u32 ctl, reset_flags = HL_DRV_RESET_HARD | HL_DRV_RESET_DELAY;
struct gaudi2_device *gaudi2 = hdev->asic_specific;
- bool reset_required = false, skip_reset = false;
+ bool reset_required = false, is_critical = false;
+ u32 ctl, reset_flags = HL_DRV_RESET_HARD;
int index, sbte_index;
u64 event_mask = 0;
u16 event_type;
@@ -8601,6 +8776,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
+ is_critical = eq_entry->ecc_data.is_critical;
break;
case GAUDI2_EVENT_TPC0_QM ... GAUDI2_EVENT_PDMA1_QM:
@@ -8626,29 +8802,30 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP:
case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP:
reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
- gaudi2_handle_qm_sei_err(hdev, event_type, &eq_entry->razwi_info);
+ gaudi2_handle_qm_sei_err(hdev, event_type, &eq_entry->razwi_info, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE:
case GAUDI2_EVENT_ROTATOR1_AXI_ERROR_RESPONSE:
index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;
- gaudi2_handle_rot_err(hdev, index, &eq_entry->razwi_with_intr_cause);
- gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
+ gaudi2_handle_rot_err(hdev, index, &eq_entry->razwi_with_intr_cause, &event_mask);
+ gaudi2_handle_qm_sei_err(hdev, event_type, NULL, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP:
index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;
gaudi2_tpc_ack_interrupts(hdev, index, "AXI_ERR_RSP",
- &eq_entry->razwi_with_intr_cause);
- gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
+ &eq_entry->razwi_with_intr_cause, &event_mask);
+ gaudi2_handle_qm_sei_err(hdev, event_type, NULL, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE:
index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;
- gaudi2_handle_dec_err(hdev, index, "AXI_ERR_RESPONSE", &eq_entry->razwi_info);
+ gaudi2_handle_dec_err(hdev, index, "AXI_ERR_RESPONSE", &eq_entry->razwi_info,
+ &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8679,7 +8856,8 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_TPC24_KERNEL_ERR:
index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) /
(GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR);
- gaudi2_tpc_ack_interrupts(hdev, index, "KRN_ERR", &eq_entry->razwi_with_intr_cause);
+ gaudi2_tpc_ack_interrupts(hdev, index, "KRN_ERR", &eq_entry->razwi_with_intr_cause,
+ &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8695,7 +8873,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_DEC9_SPI:
index = (event_type - GAUDI2_EVENT_DEC0_SPI) /
(GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI);
- gaudi2_handle_dec_err(hdev, index, "SPI", &eq_entry->razwi_info);
+ gaudi2_handle_dec_err(hdev, index, "SPI", &eq_entry->razwi_info, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8707,8 +8885,8 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -
GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE);
gaudi2_handle_mme_err(hdev, index,
- "CTRL_AXI_ERROR_RESPONSE", &eq_entry->razwi_info);
- gaudi2_handle_qm_sei_err(hdev, event_type, NULL);
+ "CTRL_AXI_ERROR_RESPONSE", &eq_entry->razwi_info, &event_mask);
+ gaudi2_handle_qm_sei_err(hdev, event_type, NULL, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8719,7 +8897,8 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) /
(GAUDI2_EVENT_MME1_QMAN_SW_ERROR -
GAUDI2_EVENT_MME0_QMAN_SW_ERROR);
- gaudi2_handle_mme_err(hdev, index, "QMAN_SW_ERROR", &eq_entry->razwi_info);
+ gaudi2_handle_mme_err(hdev, index, "QMAN_SW_ERROR", &eq_entry->razwi_info,
+ &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8730,7 +8909,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) /
(GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID -
GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID);
- gaudi2_handle_mme_wap_err(hdev, index, &eq_entry->razwi_info);
+ gaudi2_handle_mme_wap_err(hdev, index, &eq_entry->razwi_info, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8749,7 +8928,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR:
gaudi2_print_pcie_addr_dec_info(hdev,
- le64_to_cpu(eq_entry->intr_cause.intr_cause_data));
+ le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask);
reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
break;
@@ -8758,7 +8937,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_HMMU_0_AXI_ERR_RSP ... GAUDI2_EVENT_HMMU_12_AXI_ERR_RSP:
case GAUDI2_EVENT_PMMU0_PAGE_FAULT_WR_PERM ... GAUDI2_EVENT_PMMU0_SECURITY_ERROR:
case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0:
- gaudi2_handle_mmu_spi_sei_err(hdev, event_type);
+ gaudi2_handle_mmu_spi_sei_err(hdev, event_type, &event_mask);
reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8778,7 +8957,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
break;
case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT:
- gaudi2_ack_psoc_razwi_event_handler(hdev);
+ gaudi2_ack_psoc_razwi_event_handler(hdev, &event_mask);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8927,7 +9106,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE:
index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE;
- skip_reset = !gaudi2_handle_sm_err(hdev, index);
+ gaudi2_handle_sm_err(hdev, index);
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
@@ -8956,13 +9135,20 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;
break;
+ case GAUDI2_EVENT_CPU_FP32_NOT_SUPPORTED:
+ event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+ is_critical = true;
+ break;
+
default:
if (gaudi2_irq_map_table[event_type].valid)
dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",
event_type);
}
- if ((gaudi2_irq_map_table[event_type].reset || reset_required) && !skip_reset)
+ if ((gaudi2_irq_map_table[event_type].reset || reset_required) &&
+ (hdev->hard_reset_on_fw_events ||
+ (hdev->asic_prop.fw_security_enabled && is_critical)))
goto reset_device;
/* Send unmask irq only for interrupts not classified as MSG */
@@ -8975,46 +9161,84 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
return;
reset_device:
- if (hdev->hard_reset_on_fw_events) {
- hl_device_reset(hdev, reset_flags);
- event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
+ if (hdev->asic_prop.fw_security_enabled && is_critical) {
+ reset_flags |= HL_DRV_RESET_BYPASS_REQ_TO_FW;
+ event_mask |= HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
} else {
- if (!gaudi2_irq_map_table[event_type].msg)
- hl_fw_unmask_irq(hdev, event_type);
+ reset_flags |= HL_DRV_RESET_DELAY;
}
+ event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;
+ hl_device_cond_reset(hdev, reset_flags, event_mask);
+}
- if (event_mask)
- hl_notifier_event_send_all(hdev, event_mask);
+static int gaudi2_memset_memory_chunk_using_edma_qm(struct hl_device *hdev,
+ struct packet_lin_dma *lin_dma_pkt, dma_addr_t pkt_dma_addr,
+ u32 hw_queue_id, u32 size, u64 addr, u32 val)
+{
+ u32 ctl, pkt_size;
+ int rc = 0;
+
+ ctl = FIELD_PREP(GAUDI2_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);
+ ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1);
+ ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1);
+ ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1);
+
+ lin_dma_pkt->ctl = cpu_to_le32(ctl);
+ lin_dma_pkt->src_addr = cpu_to_le64(val);
+ lin_dma_pkt->dst_addr = cpu_to_le64(addr);
+ lin_dma_pkt->tsize = cpu_to_le32(size);
+
+ pkt_size = sizeof(struct packet_lin_dma);
+
+ rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, pkt_dma_addr);
+ if (rc)
+ dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n",
+ hw_queue_id);
+
+ return rc;
}
static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val)
{
- struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u32 edma_queues_id[] = {GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0,
+ GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0,
+ GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0,
+ GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0};
+ u32 chunk_size, dcore, edma_idx, sob_offset, sob_addr, comp_val,
+ old_mmubp, mmubp, num_of_pkts, busy, pkt_size;
u64 comp_addr, cur_addr = addr, end_addr = addr + size;
- u32 chunk_size, busy, dcore, edma_idx, sob_offset, sob_addr, comp_val, edma_commit;
- u32 old_mmubp, mmubp;
- int rc = 0;
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ void *lin_dma_pkts_arr;
+ dma_addr_t pkt_dma_addr;
+ int rc = 0, dma_num = 0;
+
+ if (prop->edma_enabled_mask == 0) {
+ dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
+ return -EIO;
+ }
sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;
sob_addr = mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + sob_offset;
comp_addr = CFG_BASE + sob_addr;
comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |
FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);
-
- edma_commit = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |
- FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1) |
- FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);
mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) |
FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1);
- if (prop->edma_enabled_mask == 0) {
- dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");
- return -EIO;
- }
+ /* Calculate how many lin dma pkts we'll need */
+ num_of_pkts = div64_u64(round_up(size, SZ_2G), SZ_2G);
+ pkt_size = sizeof(struct packet_lin_dma);
+
+ lin_dma_pkts_arr = hl_asic_dma_alloc_coherent(hdev, pkt_size * num_of_pkts,
+ &pkt_dma_addr, GFP_KERNEL);
+ if (!lin_dma_pkts_arr)
+ return -ENOMEM;
/*
* set mmu bypass for the scrubbing - all ddmas are configured the same so save
* only the first one to restore later
+ * also set the sob addr for all edma cores for completion.
+ * set QM as trusted to allow it to access physical address with MMU bp.
*/
old_mmubp = RREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP);
for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
@@ -9027,17 +9251,22 @@ static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 siz
WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP +
edma_offset, mmubp);
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset,
+ lower_32_bits(comp_addr));
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset,
+ upper_32_bits(comp_addr));
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset,
+ comp_val);
+ gaudi2_qman_set_test_mode(hdev,
+ edma_queues_id[dcore] + 4 * edma_idx, true);
}
}
- while (cur_addr < end_addr) {
- int dma_num = 0;
+ WREG32(sob_addr, 0);
- WREG32(sob_addr, 0);
+ while (cur_addr < end_addr) {
for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
for (edma_idx = 0 ; edma_idx < NUM_OF_EDMA_PER_DCORE ; edma_idx++) {
- u32 edma_offset = dcore * DCORE_OFFSET +
- edma_idx * DCORE_EDMA_OFFSET;
u32 edma_bit = dcore * NUM_OF_EDMA_PER_DCORE + edma_idx;
if (!(prop->edma_enabled_mask & BIT(edma_bit)))
@@ -9045,41 +9274,26 @@ static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 siz
chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr);
- WREG32(mmDCORE0_EDMA0_CORE_CTX_SRC_BASE_LO + edma_offset,
- lower_32_bits(val));
- WREG32(mmDCORE0_EDMA0_CORE_CTX_SRC_BASE_HI + edma_offset,
- upper_32_bits(val));
-
- WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_BASE_LO + edma_offset,
- lower_32_bits(cur_addr));
- WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_BASE_HI + edma_offset,
- upper_32_bits(cur_addr));
-
- WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset,
- lower_32_bits(comp_addr));
- WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset,
- upper_32_bits(comp_addr));
- WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset,
- comp_val);
-
- WREG32(mmDCORE0_EDMA0_CORE_CTX_DST_TSIZE_0 + edma_offset,
- chunk_size);
- WREG32(mmDCORE0_EDMA0_CORE_CTX_COMMIT + edma_offset, edma_commit);
+ rc = gaudi2_memset_memory_chunk_using_edma_qm(hdev,
+ (struct packet_lin_dma *)lin_dma_pkts_arr + dma_num,
+ pkt_dma_addr + dma_num * pkt_size,
+ edma_queues_id[dcore] + edma_idx * 4,
+ chunk_size, cur_addr, val);
+ if (rc)
+ goto end;
dma_num++;
-
cur_addr += chunk_size;
-
if (cur_addr == end_addr)
- goto poll;
+ break;
}
}
-poll:
- rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
- if (rc) {
- dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
- goto end;
- }
+ }
+
+ rc = hl_poll_timeout(hdev, sob_addr, busy, (busy == dma_num), 1000, 1000000);
+ if (rc) {
+ dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n");
+ goto end;
}
end:
for (dcore = 0 ; dcore < NUM_OF_DCORES ; dcore++) {
@@ -9091,10 +9305,17 @@ end:
continue;
WREG32(mmDCORE0_EDMA0_CORE_CTX_AXUSER_HB_MMU_BP + edma_offset, old_mmubp);
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_LO + edma_offset, 0);
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_ADDR_HI + edma_offset, 0);
+ WREG32(mmDCORE0_EDMA0_CORE_CTX_WR_COMP_WDATA + edma_offset, 0);
+ gaudi2_qman_set_test_mode(hdev,
+ edma_queues_id[dcore] + 4 * edma_idx, false);
}
}
WREG32(sob_addr, 0);
+ hl_asic_dma_free_coherent(hdev, pkt_size * num_of_pkts, lin_dma_pkts_arr, pkt_dma_addr);
+
return rc;
}
@@ -9165,6 +9386,7 @@ static void gaudi2_restore_user_sm_registers(struct hl_device *hdev)
gaudi2_memset_device_lbw(hdev, cq_lbw_data_addr, size, 0);
gaudi2_memset_device_lbw(hdev, cq_base_l_addr, size, 0);
gaudi2_memset_device_lbw(hdev, cq_base_h_addr, size, 0);
+ gaudi2_memset_device_lbw(hdev, cq_size_addr, size, 0);
cq_lbw_l_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0 + DCORE_OFFSET;
cq_lbw_h_addr = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 + DCORE_OFFSET;
@@ -9990,7 +10212,7 @@ static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id)
if (gaudi2_get_mmu_base(hdev, mmu_id, &mmu_base))
return;
- gaudi2_handle_page_error(hdev, mmu_base, is_pmmu);
+ gaudi2_handle_page_error(hdev, mmu_base, is_pmmu, NULL);
gaudi2_handle_access_error(hdev, mmu_base, is_pmmu);
}
@@ -10141,10 +10363,9 @@ int gaudi2_send_device_activity(struct hl_device *hdev, bool open)
{
struct gaudi2_device *gaudi2 = hdev->asic_specific;
- if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) || hdev->fw_major_version < 37)
+ if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- /* TODO: add check for FW version using minor ver once it's known */
return hl_fw_send_device_activity(hdev, open);
}