From b01ecceaf2c0c4b3f2d24aa0adcf096ab1648253 Mon Sep 17 00:00:00 2001 From: Yash Shah Date: Tue, 10 Dec 2019 16:41:09 +0530 Subject: genirq: Introduce irq_domain_translate_onecell Add a new function irq_domain_translate_onecell() that is to be used as the translate function in struct irq_domain_ops. Signed-off-by: Yash Shah Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/1575976274-13487-2-git-send-email-yash.shah@sifive.com --- include/linux/irqdomain.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 3c340dbc5a1f..698749f42ced 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -427,6 +427,11 @@ int irq_domain_translate_twocell(struct irq_domain *d, unsigned long *out_hwirq, unsigned int *out_type); +int irq_domain_translate_onecell(struct irq_domain *d, + struct irq_fwspec *fwspec, + unsigned long *out_hwirq, + unsigned int *out_type); + /* IPI functions */ int irq_reserve_ipi(struct irq_domain *domain, const struct cpumask *dest); int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest); -- cgit v1.2.3 From 5350a237b4525ad12170f16239c9e9c7797df02f Mon Sep 17 00:00:00 2001 From: Eddie James Date: Wed, 15 Jan 2020 15:29:39 -0600 Subject: dt-bindings: interrupt-controller: Add Aspeed SCU interrupt controller Document the Aspeed SCU interrupt controller and add an include file for the interrupts it provides. Signed-off-by: Eddie James Signed-off-by: Marc Zyngier Reviewed-by: Rob Herring Acked-by: Andrew Jeffery Link: https://lore.kernel.org/r/1579123790-6894-2-git-send-email-eajames@linux.ibm.com --- .../interrupt-controller/aspeed,ast2xxx-scu-ic.txt | 23 ++++++++++++++++++++++ MAINTAINERS | 7 +++++++ .../interrupt-controller/aspeed-scu-ic.h | 23 ++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt create mode 100644 include/dt-bindings/interrupt-controller/aspeed-scu-ic.h (limited to 'include') diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt new file mode 100644 index 000000000000..251ed44171db --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt @@ -0,0 +1,23 @@ +Aspeed AST25XX and AST26XX SCU Interrupt Controller + +Required Properties: + - #interrupt-cells : must be 1 + - compatible : must be "aspeed,ast2500-scu-ic", + "aspeed,ast2600-scu-ic0" or + "aspeed,ast2600-scu-ic1" + - interrupts : interrupt from the parent controller + - interrupt-controller : indicates that the controller receives and + fires new interrupts for child busses + +Example: + + syscon@1e6e2000 { + ranges = <0 0x1e6e2000 0x1a8>; + + scu_ic: interrupt-controller@18 { + #interrupt-cells = <1>; + compatible = "aspeed,ast2500-scu-ic"; + interrupts = <21>; + interrupt-controller; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index e09bd92a1e44..f9f6e4060a92 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2692,6 +2692,13 @@ S: Maintained F: drivers/pinctrl/aspeed/ F: Documentation/devicetree/bindings/pinctrl/aspeed,* +ASPEED SCU INTERRUPT CONTROLLER DRIVER +M: Eddie James +L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt +F: include/dt-bindings/interrupt-controller/aspeed-scu-ic.h + ASPEED VIDEO ENGINE DRIVER M: Eddie James L: linux-media@vger.kernel.org diff --git a/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h b/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h new file mode 100644 index 000000000000..f315d5a7f5ee --- /dev/null +++ b/include/dt-bindings/interrupt-controller/aspeed-scu-ic.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _DT_BINDINGS_INTERRUPT_CONTROLLER_ASPEED_SCU_IC_H_ +#define _DT_BINDINGS_INTERRUPT_CONTROLLER_ASPEED_SCU_IC_H_ + +#define ASPEED_SCU_IC_VGA_CURSOR_CHANGE 0 +#define ASPEED_SCU_IC_VGA_SCRATCH_REG_CHANGE 1 + +#define ASPEED_AST2500_SCU_IC_PCIE_RESET_LO_TO_HI 2 +#define ASPEED_AST2500_SCU_IC_PCIE_RESET_HI_TO_LO 3 +#define ASPEED_AST2500_SCU_IC_LPC_RESET_LO_TO_HI 4 +#define ASPEED_AST2500_SCU_IC_LPC_RESET_HI_TO_LO 5 +#define ASPEED_AST2500_SCU_IC_ISSUE_MSI 6 + +#define ASPEED_AST2600_SCU_IC0_PCIE_PERST_LO_TO_HI 2 +#define ASPEED_AST2600_SCU_IC0_PCIE_PERST_HI_TO_LO 3 +#define ASPEED_AST2600_SCU_IC0_PCIE_RCRST_LO_TO_HI 4 +#define ASPEED_AST2600_SCU_IC0_PCIE_RCRST_HI_TO_LO 5 + +#define ASPEED_AST2600_SCU_IC1_LPC_RESET_LO_TO_HI 0 +#define ASPEED_AST2600_SCU_IC1_LPC_RESET_HI_TO_LO 1 + +#endif /* _DT_BINDINGS_INTERRUPT_CONTROLLER_ASPEED_SCU_IC_H_ */ -- cgit v1.2.3 From b25319d279b63781b972c4966b4082193e69afac Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:24 +0000 Subject: irqchip/gic-v3: Detect GICv4.1 supporting RVPEID GICv4.1 supports the RVPEID ("Residency per vPE ID"), which allows for a much efficient way of making virtual CPUs resident (to allow direct injection of interrupts). The functionnality needs to be discovered on each and every redistributor in the system, and disabled if the settings are inconsistent. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-2-maz@kernel.org --- drivers/irqchip/irq-gic-v3.c | 21 ++++++++++++++++++--- include/linux/irqchip/arm-gic-v3.h | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index d6218012097b..ffcb018395ed 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -858,8 +858,21 @@ static int __gic_update_rdist_properties(struct redist_region *region, void __iomem *ptr) { u64 typer = gic_read_typer(ptr + GICR_TYPER); + gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); - gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS); + + /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ + gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); + gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | + gic_data.rdists.has_rvpeid); + + /* Detect non-sensical configurations */ + if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { + gic_data.rdists.has_direct_lpi = false; + gic_data.rdists.has_vlpis = false; + gic_data.rdists.has_rvpeid = false; + } + gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); return 1; @@ -872,9 +885,10 @@ static void gic_update_rdist_properties(void) if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) gic_data.ppi_nr = 0; pr_info("%d PPIs implemented\n", gic_data.ppi_nr); - pr_info("%sVLPI support, %sdirect LPI support\n", + pr_info("%sVLPI support, %sdirect LPI support, %sRVPEID support\n", !gic_data.rdists.has_vlpis ? "no " : "", - !gic_data.rdists.has_direct_lpi ? "no " : ""); + !gic_data.rdists.has_direct_lpi ? "no " : "", + !gic_data.rdists.has_rvpeid ? "no " : ""); } /* Check whether it's single security state view */ @@ -1566,6 +1580,7 @@ static int __init gic_init_bases(void __iomem *dist_base, &gic_data); irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); + gic_data.rdists.has_rvpeid = true; gic_data.rdists.has_vlpis = true; gic_data.rdists.has_direct_lpi = true; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index de991d6633a5..9a5f85d30701 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -234,6 +234,7 @@ #define GICR_TYPER_VLPIS (1U << 1) #define GICR_TYPER_DirectLPIS (1U << 3) #define GICR_TYPER_LAST (1U << 4) +#define GICR_TYPER_RVPEID (1U << 7) #define GIC_V3_REDIST_SIZE 0x20000 @@ -615,6 +616,7 @@ struct rdists { u64 flags; u32 gicd_typer; bool has_vlpis; + bool has_rvpeid; bool has_direct_lpi; }; -- cgit v1.2.3 From f2d834092ee276610ccb6637e5109b61fc79ab89 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:25 +0000 Subject: irqchip/gic-v3: Add GICv4.1 VPEID size discovery While GICv4.0 mandates 16 bit worth of VPEIDs, GICv4.1 allows smaller implementations to be built. Add the required glue to dynamically compute the limit. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-3-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 11 ++++++++++- drivers/irqchip/irq-gic-v3.c | 3 +++ include/linux/irqchip/arm-gic-v3.h | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index b704214390c0..6b0a8d6b53a3 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -121,7 +121,16 @@ struct its_node { #define ITS_ITT_ALIGN SZ_256 /* The maximum number of VPEID bits supported by VLPI commands */ -#define ITS_MAX_VPEID_BITS (16) +#define ITS_MAX_VPEID_BITS \ + ({ \ + int nvpeid = 16; \ + if (gic_rdists->has_rvpeid && \ + gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \ + nvpeid = 1 + (gic_rdists->gicd_typer2 & \ + GICD_TYPER2_VID); \ + \ + nvpeid; \ + }) #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) /* Convert page order to size in bytes */ diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index ffcb018395ed..286f98222878 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1576,6 +1576,9 @@ static int __init gic_init_bases(void __iomem *dist_base, pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); + + gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); + gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, &gic_data); irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 9a5f85d30701..9dfe64189d99 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -13,6 +13,7 @@ #define GICD_CTLR 0x0000 #define GICD_TYPER 0x0004 #define GICD_IIDR 0x0008 +#define GICD_TYPER2 0x000C #define GICD_STATUSR 0x0010 #define GICD_SETSPI_NSR 0x0040 #define GICD_CLRSPI_NSR 0x0048 @@ -89,6 +90,9 @@ #define GICD_TYPER_ESPIS(typer) \ (((typer) & GICD_TYPER_ESPI) ? GICD_TYPER_SPIS((typer) >> 27) : 0) +#define GICD_TYPER2_VIL (1U << 7) +#define GICD_TYPER2_VID GENMASK(4, 0) + #define GICD_IROUTER_SPI_MODE_ONE (0U << 31) #define GICD_IROUTER_SPI_MODE_ANY (1U << 31) @@ -615,6 +619,7 @@ struct rdists { void *prop_table_va; u64 flags; u32 gicd_typer; + u32 gicd_typer2; bool has_vlpis; bool has_rvpeid; bool has_direct_lpi; -- cgit v1.2.3 From 5e5168461c22c8738d31d4ee12a5cbc2ab0aa440 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:28 +0000 Subject: irqchip/gic-v4.1: VPE table (aka GICR_VPROPBASER) allocation GICv4.1 defines a new VPE table that is potentially shared between both the ITSs and the redistributors, following complicated affinity rules. To make things more confusing, the programming of this table at the redistributor level is reusing the GICv4.0 GICR_VPROPBASER register for something completely different. The code flow is somewhat complexified by the need to respect the affinities required by the HW, meaning that tables can either be inherited from a previously discovered ITS or redistributor. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-6-maz@kernel.org --- arch/arm/include/asm/arch_gicv3.h | 2 + arch/arm64/include/asm/arch_gicv3.h | 1 + drivers/irqchip/irq-gic-v3-its.c | 310 +++++++++++++++++++++++++++++++++++- include/linux/irqchip/arm-gic-v3.h | 33 +++- 4 files changed, 339 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index fa50bb04f580..b5752f0e8936 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -10,6 +10,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -327,6 +328,7 @@ static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr) /* * GITS_VPROPBASER - hi and lo bits may be accessed independently. */ +#define gits_read_vpropbaser(c) __gic_readq_nonatomic(c) #define gits_write_vpropbaser(v, c) __gic_writeq_nonatomic(v, c) /* diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index 89e4c8b79349..4750fc8030c3 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -141,6 +141,7 @@ static inline u32 gic_read_rpr(void) #define gicr_read_pendbaser(c) readq_relaxed(c) #define gits_write_vpropbaser(v, c) writeq_relaxed(v, c) +#define gits_read_vpropbaser(c) readq_relaxed(c) #define gits_write_vpendbaser(v, c) writeq_relaxed(v, c) #define gits_read_vpendbaser(c) readq_relaxed(c) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 6b0a8d6b53a3..9124f7f45dd3 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -106,6 +106,7 @@ struct its_node { u64 typer; u64 cbaser_save; u32 ctlr_save; + u32 mpidr; struct list_head its_device_list; u64 flags; unsigned long list_nr; @@ -116,6 +117,7 @@ struct its_node { }; #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) +#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) #define ITS_ITT_ALIGN SZ_256 @@ -2100,6 +2102,65 @@ static bool its_parse_indirect_baser(struct its_node *its, return indirect; } +static u32 compute_common_aff(u64 val) +{ + u32 aff, clpiaff; + + aff = FIELD_GET(GICR_TYPER_AFFINITY, val); + clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val); + + return aff & ~(GENMASK(31, 0) >> (clpiaff * 8)); +} + +static u32 compute_its_aff(struct its_node *its) +{ + u64 val; + u32 svpet; + + /* + * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute + * the resulting affinity. We then use that to see if this match + * our own affinity. + */ + svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); + val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet); + val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); + return compute_common_aff(val); +} + +static struct its_node *find_sibling_its(struct its_node *cur_its) +{ + struct its_node *its; + u32 aff; + + if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) + return NULL; + + aff = compute_its_aff(cur_its); + + list_for_each_entry(its, &its_nodes, entry) { + u64 baser; + + if (!is_v4_1(its) || its == cur_its) + continue; + + if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) + continue; + + if (aff != compute_its_aff(its)) + continue; + + /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ + baser = its->tables[2].val; + if (!(baser & GITS_BASER_VALID)) + continue; + + return its; + } + + return NULL; +} + static void its_free_tables(struct its_node *its) { int i; @@ -2142,6 +2203,17 @@ static int its_alloc_tables(struct its_node *its) break; case GITS_BASER_TYPE_VCPU: + if (is_v4_1(its)) { + struct its_node *sibling; + + WARN_ON(i != 2); + if ((sibling = find_sibling_its(its))) { + *baser = sibling->tables[2]; + its_write_baser(its, baser, baser->val); + continue; + } + } + indirect = its_parse_indirect_baser(its, baser, psz, &order, ITS_MAX_VPEID_BITS); @@ -2163,6 +2235,220 @@ static int its_alloc_tables(struct its_node *its) return 0; } +static u64 inherit_vpe_l1_table_from_its(void) +{ + struct its_node *its; + u64 val; + u32 aff; + + val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); + aff = compute_common_aff(val); + + list_for_each_entry(its, &its_nodes, entry) { + u64 baser, addr; + + if (!is_v4_1(its)) + continue; + + if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) + continue; + + if (aff != compute_its_aff(its)) + continue; + + /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ + baser = its->tables[2].val; + if (!(baser & GITS_BASER_VALID)) + continue; + + /* We have a winner! */ + val = GICR_VPROPBASER_4_1_VALID; + if (baser & GITS_BASER_INDIRECT) + val |= GICR_VPROPBASER_4_1_INDIRECT; + val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, + FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)); + switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) { + case GIC_PAGE_SIZE_64K: + addr = GITS_BASER_ADDR_48_to_52(baser); + break; + default: + addr = baser & GENMASK_ULL(47, 12); + break; + } + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12); + val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, + FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); + val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, + FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); + val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); + + return val; + } + + return 0; +} + +static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) +{ + u32 aff; + u64 val; + int cpu; + + val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); + aff = compute_common_aff(val); + + for_each_possible_cpu(cpu) { + void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; + u32 tmp; + + if (!base || cpu == smp_processor_id()) + continue; + + val = gic_read_typer(base + GICR_TYPER); + tmp = compute_common_aff(val); + if (tmp != aff) + continue; + + /* + * At this point, we have a victim. This particular CPU + * has already booted, and has an affinity that matches + * ours wrt CommonLPIAff. Let's use its own VPROPBASER. + * Make sure we don't write the Z bit in that case. + */ + val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); + val &= ~GICR_VPROPBASER_4_1_Z; + + *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; + + return val; + } + + return 0; +} + +static int allocate_vpe_l1_table(void) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val, gpsz, npg, pa; + unsigned int psz = SZ_64K; + unsigned int np, epp, esz; + struct page *page; + + if (!gic_rdists->has_rvpeid) + return 0; + + /* + * if VPENDBASER.Valid is set, disable any previously programmed + * VPE by setting PendingLast while clearing Valid. This has the + * effect of making sure no doorbell will be generated and we can + * then safely clear VPROPBASER.Valid. + */ + if (gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) + gits_write_vpendbaser(GICR_VPENDBASER_PendingLast, + vlpi_base + GICR_VPENDBASER); + + /* + * If we can inherit the configuration from another RD, let's do + * so. Otherwise, we have to go through the allocation process. We + * assume that all RDs have the exact same requirements, as + * nothing will work otherwise. + */ + val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); + if (val & GICR_VPROPBASER_4_1_VALID) + goto out; + + gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!gic_data_rdist()->vpe_table_mask) + return -ENOMEM; + + val = inherit_vpe_l1_table_from_its(); + if (val & GICR_VPROPBASER_4_1_VALID) + goto out; + + /* First probe the page size */ + val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); + gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + val = gits_read_vpropbaser(vlpi_base + GICR_VPROPBASER); + gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); + esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); + + switch (gpsz) { + default: + gpsz = GIC_PAGE_SIZE_4K; + /* fall through */ + case GIC_PAGE_SIZE_4K: + psz = SZ_4K; + break; + case GIC_PAGE_SIZE_16K: + psz = SZ_16K; + break; + case GIC_PAGE_SIZE_64K: + psz = SZ_64K; + break; + } + + /* + * Start populating the register from scratch, including RO fields + * (which we want to print in debug cases...) + */ + val = 0; + val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz); + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz); + + /* How many entries per GIC page? */ + esz++; + epp = psz / (esz * SZ_8); + + /* + * If we need more than just a single L1 page, flag the table + * as indirect and compute the number of required L1 pages. + */ + if (epp < ITS_MAX_VPEID) { + int nl2; + + val |= GICR_VPROPBASER_4_1_INDIRECT; + + /* Number of L2 pages required to cover the VPEID space */ + nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp); + + /* Number of L1 pages to point to the L2 pages */ + npg = DIV_ROUND_UP(nl2 * SZ_8, psz); + } else { + npg = 1; + } + + val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg); + + /* Right, that's the number of CPU pages we need for L1 */ + np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); + + pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", + np, npg, psz, epp, esz); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(np * PAGE_SIZE)); + if (!page) + return -ENOMEM; + + gic_data_rdist()->vpe_l1_page = page; + pa = virt_to_phys(page_address(page)); + WARN_ON(!IS_ALIGNED(pa, psz)); + + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12); + val |= GICR_VPROPBASER_RaWb; + val |= GICR_VPROPBASER_InnerShareable; + val |= GICR_VPROPBASER_4_1_Z; + val |= GICR_VPROPBASER_4_1_VALID; + +out: + gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); + + pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", + smp_processor_id(), val, + cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); + + return 0; +} + static int its_alloc_collections(struct its_node *its) { int i; @@ -2362,7 +2648,7 @@ static void its_cpu_init_lpis(void) val |= GICR_CTLR_ENABLE_LPIS; writel_relaxed(val, rbase + GICR_CTLR); - if (gic_rdists->has_vlpis) { + if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); /* @@ -2386,6 +2672,16 @@ static void its_cpu_init_lpis(void) WARN_ON(val & GICR_VPENDBASER_Dirty); } + if (allocate_vpe_l1_table()) { + /* + * If the allocation has failed, we're in massive trouble. + * Disable direct injection, and pray that no VM was + * already running... + */ + gic_rdists->has_rvpeid = false; + gic_rdists->has_vlpis = false; + } + /* Make sure the GIC has seen the above */ dsb(sy); out: @@ -3788,6 +4084,14 @@ static int __init its_probe_one(struct resource *res, } else { pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); } + + if (is_v4_1(its)) { + u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); + its->mpidr = readl_relaxed(its_base + GITS_MPIDR); + + pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", + &res->start, its->mpidr, svpet); + } } its->numa_node = numa_node; @@ -4148,6 +4452,8 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, bool has_v4 = false; int err; + gic_rdists = rdists; + its_parent = parent_domain; of_node = to_of_node(handle); if (of_node) @@ -4160,8 +4466,6 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, return -ENXIO; } - gic_rdists = rdists; - err = allocate_lpi_tables(); if (err) return err; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 9dfe64189d99..db0a11193d92 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -102,6 +102,11 @@ #define GIC_V3_DIST_SIZE 0x10000 +#define GIC_PAGE_SIZE_4K 0ULL +#define GIC_PAGE_SIZE_16K 1ULL +#define GIC_PAGE_SIZE_64K 2ULL +#define GIC_PAGE_SIZE_MASK 3ULL + /* * Re-Distributor registers, offsets from RD_base */ @@ -239,6 +244,8 @@ #define GICR_TYPER_DirectLPIS (1U << 3) #define GICR_TYPER_LAST (1U << 4) #define GICR_TYPER_RVPEID (1U << 7) +#define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) +#define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) #define GIC_V3_REDIST_SIZE 0x20000 @@ -277,6 +284,18 @@ #define GICR_VPROPBASER_RaWaWt GIC_BASER_CACHEABILITY(GICR_VPROPBASER, INNER, RaWaWt) #define GICR_VPROPBASER_RaWaWb GIC_BASER_CACHEABILITY(GICR_VPROPBASER, INNER, RaWaWb) +/* + * GICv4.1 VPROPBASER reinvention. A subtle mix between the old + * VPROPBASER and ITS_BASER. Just not quite any of the two. + */ +#define GICR_VPROPBASER_4_1_VALID (1ULL << 63) +#define GICR_VPROPBASER_4_1_ENTRY_SIZE GENMASK_ULL(61, 59) +#define GICR_VPROPBASER_4_1_INDIRECT (1ULL << 55) +#define GICR_VPROPBASER_4_1_PAGE_SIZE GENMASK_ULL(54, 53) +#define GICR_VPROPBASER_4_1_Z (1ULL << 52) +#define GICR_VPROPBASER_4_1_ADDR GENMASK_ULL(51, 12) +#define GICR_VPROPBASER_4_1_SIZE GENMASK_ULL(6, 0) + #define GICR_VPENDBASER 0x0078 #define GICR_VPENDBASER_SHAREABILITY_SHIFT (10) @@ -314,6 +333,7 @@ #define GITS_CTLR 0x0000 #define GITS_IIDR 0x0004 #define GITS_TYPER 0x0008 +#define GITS_MPIDR 0x0018 #define GITS_CBASER 0x0080 #define GITS_CWRITER 0x0088 #define GITS_CREADR 0x0090 @@ -347,6 +367,8 @@ #define GITS_TYPER_HCC_SHIFT 24 #define GITS_TYPER_HCC(r) (((r) >> GITS_TYPER_HCC_SHIFT) & 0xff) #define GITS_TYPER_VMOVP (1ULL << 37) +#define GITS_TYPER_VMAPP (1ULL << 40) +#define GITS_TYPER_SVPET GENMASK_ULL(42, 41) #define GITS_IIDR_REV_SHIFT 12 #define GITS_IIDR_REV_MASK (0xf << GITS_IIDR_REV_SHIFT) @@ -417,10 +439,11 @@ #define GITS_BASER_InnerShareable \ GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) #define GITS_BASER_PAGE_SIZE_SHIFT (8) -#define GITS_BASER_PAGE_SIZE_4K (0ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_16K (1ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_64K (2ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_MASK (3ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define __GITS_BASER_PSZ(sz) (GIC_PAGE_SIZE_ ## sz << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_4K __GITS_BASER_PSZ(4K) +#define GITS_BASER_PAGE_SIZE_16K __GITS_BASER_PSZ(16K) +#define GITS_BASER_PAGE_SIZE_64K __GITS_BASER_PSZ(64K) +#define GITS_BASER_PAGE_SIZE_MASK __GITS_BASER_PSZ(MASK) #define GITS_BASER_PAGES_MAX 256 #define GITS_BASER_PAGES_SHIFT (0) #define GITS_BASER_NR_PAGES(r) (((r) & 0xff) + 1) @@ -612,8 +635,10 @@ struct rdists { struct { void __iomem *rd_base; struct page *pend_page; + struct page *vpe_l1_page; phys_addr_t phys_base; bool lpi_enabled; + cpumask_t *vpe_table_mask; } __percpu *rdist; phys_addr_t prop_table_pa; void *prop_table_va; -- cgit v1.2.3 From 64edfaa9a2342a3ce34f8cb982c2c2df84db4de3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:29 +0000 Subject: irqchip/gic-v4.1: Implement the v4.1 flavour of VMAPP The ITS VMAPP command gains some new fields with GICv4.1: - a default doorbell, which allows a single doorbell to be used for all the VLPIs routed to a given VPE - a pointer to the configuration table (instead of having it in a register that gets context switched) - a flag indicating whether this is the first map or the last unmap for this particular VPE - a flag indicating whether the pending table is known to be zeroed, or not Plumb in the new fields in the VMAPP builder, and add the map/unmap refcounting so that the ITS can do the right thing. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-7-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 60 ++++++++++++++++++++++++++++++++++---- include/linux/irqchip/arm-gic-v4.h | 18 +++++++++--- 2 files changed, 69 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 9124f7f45dd3..fbf4ca796b8a 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -449,6 +449,27 @@ static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); } +static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa) +{ + its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); +} + +static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc) +{ + its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); +} + +static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz) +{ + its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); +} + +static void its_encode_vmapp_default_db(struct its_cmd_block *cmd, + u32 vpe_db_lpi) +{ + its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); +} + static inline void its_fixup_cmd(struct its_cmd_block *cmd) { /* Let's fixup BE commands */ @@ -632,19 +653,45 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, struct its_cmd_block *cmd, struct its_cmd_desc *desc) { - unsigned long vpt_addr; + unsigned long vpt_addr, vconf_addr; u64 target; - - vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); - target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; + bool alloc; its_encode_cmd(cmd, GITS_CMD_VMAPP); its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); its_encode_valid(cmd, desc->its_vmapp_cmd.valid); + + if (!desc->its_vmapp_cmd.valid) { + if (is_v4_1(its)) { + alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); + its_encode_alloc(cmd, alloc); + } + + goto out; + } + + vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); + target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; + its_encode_target(cmd, target); its_encode_vpt_addr(cmd, vpt_addr); its_encode_vpt_size(cmd, LPI_NRBITS - 1); + if (!is_v4_1(its)) + goto out; + + vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); + + alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); + + its_encode_alloc(cmd, alloc); + + /* We can only signal PTZ when alloc==1. Why do we have two bits? */ + its_encode_ptz(cmd, alloc); + its_encode_vconf_addr(cmd, vconf_addr); + its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); + +out: its_fixup_cmd(cmd); return valid_vpe(its, desc->its_vmapp_cmd.vpe); @@ -3492,7 +3539,10 @@ static int its_vpe_init(struct its_vpe *vpe) vpe->vpe_id = vpe_id; vpe->vpt_page = vpt_page; - vpe->vpe_proxy_event = -1; + if (gic_rdists->has_rvpeid) + atomic_set(&vpe->vmapp_count, 0); + else + vpe->vpe_proxy_event = -1; return 0; } diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 5dbcfc65f21e..498e523085a7 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -39,8 +39,20 @@ struct its_vpe { irq_hw_number_t vpe_db_lpi; /* VPE resident */ bool resident; - /* VPE proxy mapping */ - int vpe_proxy_event; + union { + /* GICv4.0 implementations */ + struct { + /* VPE proxy mapping */ + int vpe_proxy_event; + /* Implementation Defined Area Invalid */ + bool idai; + }; + /* GICv4.1 implementations */ + struct { + atomic_t vmapp_count; + }; + }; + /* * This collection ID is used to indirect the target * redistributor for this VPE. The ID itself isn't involved in @@ -49,8 +61,6 @@ struct its_vpe { u16 col_idx; /* Unique (system-wide) VPE identifier */ u16 vpe_id; - /* Implementation Defined Area Invalid */ - bool idai; /* Pending VLPIs on schedule out? */ bool pending_last; }; -- cgit v1.2.3 From d97c97baa214486cc3d64c996a2214475f6cc83c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:33 +0000 Subject: irqchip/gic-v4.1: Add mask/unmask doorbell callbacks masking/unmasking doorbells on GICv4.1 relies on a new INVDB command, which broadcasts the invalidation to all RDs. Implement the new command as well as the masking callbacks, and plug the whole thing into the v4.1 VPE irqchip. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-11-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 73 ++++++++++++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 3 +- 2 files changed, 75 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2b477e29eb4c..5ef706e8fc1f 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -333,6 +333,10 @@ struct its_cmd_desc { u16 seq_num; u16 its_list; } its_vmovp_cmd; + + struct { + struct its_vpe *vpe; + } its_invdb_cmd; }; }; @@ -831,6 +835,21 @@ static struct its_vpe *its_build_vclear_cmd(struct its_node *its, return valid_vpe(its, map->vpe); } +static struct its_vpe *its_build_invdb_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + if (WARN_ON(!is_v4_1(its))) + return NULL; + + its_encode_cmd(cmd, GITS_CMD_INVDB); + its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); + + its_fixup_cmd(cmd); + + return valid_vpe(its, desc->its_invdb_cmd.vpe); +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -1239,6 +1258,14 @@ static void its_send_vclear(struct its_device *dev, u32 event_id) its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); } +static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) +{ + struct its_cmd_desc desc; + + desc.its_invdb_cmd.vpe = vpe; + its_send_single_vcommand(its, its_build_invdb_cmd, &desc); +} + /* * irqchip functions - assumes MSI, mostly. */ @@ -3553,6 +3580,50 @@ static struct irq_chip its_vpe_irq_chip = { .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, }; +static struct its_node *find_4_1_its(void) +{ + static struct its_node *its = NULL; + + if (!its) { + list_for_each_entry(its, &its_nodes, entry) { + if (is_v4_1(its)) + return its; + } + + /* Oops? */ + its = NULL; + } + + return its; +} + +static void its_vpe_4_1_send_inv(struct irq_data *d) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + struct its_node *its; + + /* + * GICv4.1 wants doorbells to be invalidated using the + * INVDB command in order to be broadcast to all RDs. Send + * it to the first valid ITS, and let the HW do its magic. + */ + its = find_4_1_its(); + if (its) + its_send_invdb(its, vpe); +} + +static void its_vpe_4_1_mask_irq(struct irq_data *d) +{ + lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); + its_vpe_4_1_send_inv(d); +} + +static void its_vpe_4_1_unmask_irq(struct irq_data *d) +{ + lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); + its_vpe_4_1_send_inv(d); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { struct its_cmd_info *info = vcpu_info; @@ -3574,6 +3645,8 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) static struct irq_chip its_vpe_4_1_irq_chip = { .name = "GICv4.1-vpe", + .irq_mask = its_vpe_4_1_mask_irq, + .irq_unmask = its_vpe_4_1_unmask_irq, .irq_eoi = irq_chip_eoi_parent, .irq_set_affinity = its_vpe_set_affinity, .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index db0a11193d92..1f17181a9f7e 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -484,8 +484,9 @@ #define GITS_CMD_VMAPTI GITS_CMD_GICv4(GITS_CMD_MAPTI) #define GITS_CMD_VMOVI GITS_CMD_GICv4(GITS_CMD_MOVI) #define GITS_CMD_VSYNC GITS_CMD_GICv4(GITS_CMD_SYNC) -/* VMOVP is the odd one, as it doesn't have a physical counterpart */ +/* VMOVP and INVDB are the odd ones, as they dont have a physical counterpart */ #define GITS_CMD_VMOVP GITS_CMD_GICv4(2) +#define GITS_CMD_INVDB GITS_CMD_GICv4(0xe) /* * ITS error numbers -- cgit v1.2.3 From 91bf6395f7b8614a5a9934a0ae9c8b5312d77b29 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:34 +0000 Subject: irqchip/gic-v4.1: Add VPE residency callback Making a VPE resident on GICv4.1 is pretty simple, as it is just a single write to the local redistributor. We just need extra information about which groups to enable, which the KVM code will have to provide. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-12-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 17 +++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 9 +++++++++ include/linux/irqchip/arm-gic-v4.h | 5 +++++ 3 files changed, 31 insertions(+) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5ef706e8fc1f..3adc597e2a82 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3624,12 +3624,29 @@ static void its_vpe_4_1_unmask_irq(struct irq_data *d) its_vpe_4_1_send_inv(d); } +static void its_vpe_4_1_schedule(struct its_vpe *vpe, + struct its_cmd_info *info) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val = 0; + + /* Schedule the VPE */ + val |= GICR_VPENDBASER_Valid; + val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; + val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; + val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); + + gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_cmd_info *info = vcpu_info; switch (info->cmd_type) { case SCHEDULE_VPE: + its_vpe_4_1_schedule(vpe, info); return 0; case DESCHEDULE_VPE: diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 1f17181a9f7e..822dae6abf75 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -327,6 +327,15 @@ #define GICR_VPENDBASER_IDAI (1ULL << 62) #define GICR_VPENDBASER_Valid (1ULL << 63) +/* + * GICv4.1 VPENDBASER, used for VPE residency. On top of these fields, + * also use the above Valid, PendingLast and Dirty. + */ +#define GICR_VPENDBASER_4_1_DB (1ULL << 62) +#define GICR_VPENDBASER_4_1_VGRP0EN (1ULL << 59) +#define GICR_VPENDBASER_4_1_VGRP1EN (1ULL << 58) +#define GICR_VPENDBASER_4_1_VPEID GENMASK_ULL(15, 0) + /* * ITS registers, offsets from ITS_base */ diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 498e523085a7..d9c34968467a 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -100,6 +100,11 @@ struct its_cmd_info { union { struct its_vlpi_map *map; u8 config; + bool req_db; + struct { + bool g0en; + bool g1en; + }; }; }; -- cgit v1.2.3 From b4a4bd0f2629ec2ece7690de1b4721529da29871 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:36 +0000 Subject: irqchip/gic-v4.1: Add VPE INVALL callback GICv4.1 redistributors have a VPE-aware INVALL register. Progress! We can now emulate a guest-requested INVALL without emiting a VINVALL command. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-14-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 14 ++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 6 ++++++ 2 files changed, 20 insertions(+) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 69b16e5da383..1d8d96a8cc26 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3669,6 +3669,19 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, } } +static void its_vpe_4_1_invall(struct its_vpe *vpe) +{ + void __iomem *rdbase; + u64 val; + + val = GICR_INVALLR_V; + val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); + + /* Target the redistributor this vPE is currently known on */ + rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; + gic_write_lpir(val, rdbase + GICR_INVALLR); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); @@ -3684,6 +3697,7 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) return 0; case INVALL_VPE: + its_vpe_4_1_invall(vpe); return 0; default: diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 822dae6abf75..49ed6fa5e293 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -247,6 +247,12 @@ #define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) #define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) +#define GICR_INVLPIR_VPEID GENMASK_ULL(47, 32) +#define GICR_INVLPIR_V GENMASK_ULL(63, 63) + +#define GICR_INVALLR_VPEID GICR_INVLPIR_VPEID +#define GICR_INVALLR_V GICR_INVLPIR_V + #define GIC_V3_REDIST_SIZE 0x20000 #define LPI_PROP_GROUP1 (1 << 1) -- cgit v1.2.3 From f4a81f5a853e0b7c38bfad3afd6d0365d654e777 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:38 +0000 Subject: irqchip/gic-v4.1: Allow direct invalidation of VLPIs Just like for INVALL, GICv4.1 has grown a VPE-aware INVLPI register. Let's plumb it in and make use of the DirectLPI code in that case. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-16-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 55 +++++++++++++++++++++++++------------- include/linux/irqchip/arm-gic-v3.h | 1 + 2 files changed, 37 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 53e91c98acb2..f71758632f8d 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -227,11 +227,27 @@ static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev, return &its_dev->event_map.vlpi_maps[event]; } -static struct its_collection *irq_to_col(struct irq_data *d) +static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) +{ + if (irqd_is_forwarded_to_vcpu(d)) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + return dev_event_to_vlpi_map(its_dev, event); + } + + return NULL; +} + +static int irq_to_cpuid(struct irq_data *d) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_vlpi_map *map = get_vlpi_map(d); + + if (map) + return map->vpe->col_idx; - return dev_event_to_col(its_dev, its_get_event_id(d)); + return its_dev->event_map.col_map[its_get_event_id(d)]; } static struct its_collection *valid_col(struct its_collection *col) @@ -1269,18 +1285,6 @@ static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) /* * irqchip functions - assumes MSI, mostly. */ -static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) -{ - if (irqd_is_forwarded_to_vcpu(d)) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - u32 event = its_get_event_id(d); - - return dev_event_to_vlpi_map(its_dev, event); - } - - return NULL; -} - static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) { struct its_vlpi_map *map = get_vlpi_map(d); @@ -1323,13 +1327,25 @@ static void wait_for_syncr(void __iomem *rdbase) static void direct_lpi_inv(struct irq_data *d) { - struct its_collection *col; + struct its_vlpi_map *map = get_vlpi_map(d); void __iomem *rdbase; + u64 val; + + if (map) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + + WARN_ON(!is_v4_1(its_dev->its)); + + val = GICR_INVLPIR_V; + val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); + val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); + } else { + val = d->hwirq; + } /* Target the redistributor this LPI is currently routed to */ - col = irq_to_col(d); - rdbase = per_cpu_ptr(gic_rdists->rdist, col->col_id)->rd_base; - gic_write_lpir(d->hwirq, rdbase + GICR_INVLPIR); + rdbase = per_cpu_ptr(gic_rdists->rdist, irq_to_cpuid(d))->rd_base; + gic_write_lpir(val, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); } @@ -1339,7 +1355,8 @@ static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) struct its_device *its_dev = irq_data_get_irq_chip_data(d); lpi_write_config(d, clr, set); - if (gic_rdists->has_direct_lpi && !irqd_is_forwarded_to_vcpu(d)) + if (gic_rdists->has_direct_lpi && + (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) direct_lpi_inv(d); else if (!irqd_is_forwarded_to_vcpu(d)) its_send_inv(its_dev, its_get_event_id(d)); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 49ed6fa5e293..f0b8ca766e7d 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -247,6 +247,7 @@ #define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) #define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) +#define GICR_INVLPIR_INTID GENMASK_ULL(31, 0) #define GICR_INVLPIR_VPEID GENMASK_ULL(47, 32) #define GICR_INVLPIR_V GENMASK_ULL(63, 63) -- cgit v1.2.3 From 11ea68f553e244851d15793a7fa33a97c46d8271 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 20 Jan 2020 17:16:25 +0800 Subject: genirq, sched/isolation: Isolate from handling managed interrupts The affinity of managed interrupts is completely handled in the kernel and cannot be changed via the /proc/irq/* interfaces from user space. As the kernel tries to spread out interrupts evenly accross CPUs on x86 to prevent vector exhaustion, it can happen that a managed interrupt whose affinity mask contains both isolated and housekeeping CPUs is routed to an isolated CPU. As a consequence IO submitted on a housekeeping CPU causes interrupts on the isolated CPU. Add a new sub-parameter 'managed_irq' for 'isolcpus' and the corresponding logic in the interrupt affinity selection code. The subparameter indicates to the interrupt affinity selection logic that it should try to avoid the above scenario. This isolation is best effort and only effective if the automatically assigned interrupt mask of a device queue contains isolated and housekeeping CPUs. If housekeeping CPUs are online then such interrupts are directed to the housekeeping CPU so that IO submitted on the housekeeping CPU cannot disturb the isolated CPU. If a queue's affinity mask contains only isolated CPUs then this parameter has no effect on the interrupt routing decision, though interrupts are only happening when tasks running on those isolated CPUs submit IO. IO submitted on housekeeping CPUs has no influence on those queues. If the affinity mask contains both housekeeping and isolated CPUs, but none of the contained housekeeping CPUs is online, then the interrupt is also routed to an isolated CPU. Interrupts are only delivered when one of the isolated CPUs in the affinity mask submits IO. If one of the contained housekeeping CPUs comes online, the CPU hotplug logic migrates the interrupt automatically back to the upcoming housekeeping CPU. Depending on the type of interrupt controller, this can require that at least one interrupt is delivered to the isolated CPU in order to complete the migration. [ tglx: Removed unused parameter, added and edited comments/documentation and rephrased the changelog so it contains more details. ] Signed-off-by: Ming Lei Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20200120091625.17912-1-ming.lei@redhat.com --- Documentation/admin-guide/kernel-parameters.txt | 26 ++++++++++++++-- include/linux/sched/isolation.h | 1 + kernel/irq/cpuhotplug.c | 21 +++++++++++-- kernel/irq/manage.c | 41 ++++++++++++++++++++++++- kernel/sched/isolation.c | 6 ++++ 5 files changed, 90 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ade4e6ec23e0..765e4274ba71 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1933,9 +1933,31 @@ begins at 0 and the maximum value is "number of CPUs in system - 1". - The format of is described above. - + managed_irq + + Isolate from being targeted by managed interrupts + which have an interrupt mask containing isolated + CPUs. The affinity of managed interrupts is + handled by the kernel and cannot be changed via + the /proc/irq/* interfaces. + + This isolation is best effort and only effective + if the automatically assigned interrupt mask of a + device queue contains isolated and housekeeping + CPUs. If housekeeping CPUs are online then such + interrupts are directed to the housekeeping CPU + so that IO submitted on the housekeeping CPU + cannot disturb the isolated CPU. + + If a queue's affinity mask contains only isolated + CPUs then this parameter has no effect on the + interrupt routing decision, though interrupts are + only delivered when tasks running on those + isolated CPUs submit IO. IO submitted on + housekeeping CPUs has no influence on those + queues. + The format of is described above. iucv= [HW,NET] diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 6c8512d3be88..0fbcbacd1b29 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -13,6 +13,7 @@ enum hk_flags { HK_FLAG_TICK = (1 << 4), HK_FLAG_DOMAIN = (1 << 5), HK_FLAG_WQ = (1 << 6), + HK_FLAG_MANAGED_IRQ = (1 << 7), }; #ifdef CONFIG_CPU_ISOLATION diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 6c7ca2e983a5..02236b13b359 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internals.h" @@ -171,6 +172,20 @@ void irq_migrate_all_off_this_cpu(void) } } +static bool hk_should_isolate(struct irq_data *data, unsigned int cpu) +{ + const struct cpumask *hk_mask; + + if (!housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) + return false; + + hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ); + if (cpumask_subset(irq_data_get_effective_affinity_mask(data), hk_mask)) + return false; + + return cpumask_test_cpu(cpu, hk_mask); +} + static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu) { struct irq_data *data = irq_desc_get_irq_data(desc); @@ -188,9 +203,11 @@ static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu) /* * If the interrupt can only be directed to a single target * CPU then it is already assigned to a CPU in the affinity - * mask. No point in trying to move it around. + * mask. No point in trying to move it around unless the + * isolation mechanism requests to move it to an upcoming + * housekeeping CPU. */ - if (!irqd_is_single_target(data)) + if (!irqd_is_single_target(data) || hk_should_isolate(data, cpu)) irq_set_affinity_locked(data, affinity, false); } diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index b6c53ab053d2..818b2802d3e7 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -217,7 +218,45 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, if (!chip || !chip->irq_set_affinity) return -EINVAL; - ret = chip->irq_set_affinity(data, mask, force); + /* + * If this is a managed interrupt and housekeeping is enabled on + * it check whether the requested affinity mask intersects with + * a housekeeping CPU. If so, then remove the isolated CPUs from + * the mask and just keep the housekeeping CPU(s). This prevents + * the affinity setter from routing the interrupt to an isolated + * CPU to avoid that I/O submitted from a housekeeping CPU causes + * interrupts on an isolated one. + * + * If the masks do not intersect or include online CPU(s) then + * keep the requested mask. The isolated target CPUs are only + * receiving interrupts when the I/O operation was submitted + * directly from them. + * + * If all housekeeping CPUs in the affinity mask are offline, the + * interrupt will be migrated by the CPU hotplug code once a + * housekeeping CPU which belongs to the affinity mask comes + * online. + */ + if (irqd_affinity_is_managed(data) && + housekeeping_enabled(HK_FLAG_MANAGED_IRQ)) { + const struct cpumask *hk_mask, *prog_mask; + + static DEFINE_RAW_SPINLOCK(tmp_mask_lock); + static struct cpumask tmp_mask; + + hk_mask = housekeeping_cpumask(HK_FLAG_MANAGED_IRQ); + + raw_spin_lock(&tmp_mask_lock); + cpumask_and(&tmp_mask, mask, hk_mask); + if (!cpumask_intersects(&tmp_mask, cpu_online_mask)) + prog_mask = mask; + else + prog_mask = &tmp_mask; + ret = chip->irq_set_affinity(data, prog_mask, force); + raw_spin_unlock(&tmp_mask_lock); + } else { + ret = chip->irq_set_affinity(data, mask, force); + } switch (ret) { case IRQ_SET_MASK_OK: case IRQ_SET_MASK_OK_DONE: diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 9fcb2a695a41..008d6ac2342b 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -163,6 +163,12 @@ static int __init housekeeping_isolcpus_setup(char *str) continue; } + if (!strncmp(str, "managed_irq,", 12)) { + str += 12; + flags |= HK_FLAG_MANAGED_IRQ; + continue; + } + pr_warn("isolcpus: Error, unknown flag\n"); return 0; } -- cgit v1.2.3