From c1726d6f1ad2f1d83e5db1e0142756e9255a82b3 Mon Sep 17 00:00:00 2001 From: Yasuaki Ishimatsu Date: Tue, 17 Jul 2007 21:21:26 +0900 Subject: [IA64] Use per iosapic lock for indirect iosapic register access Use per-iosapic lock for indirect iosapic register access. It reduces lock contention. Signed-off-by: Kenji Kaneshige Signed-off-by: Yasuaki Ishimatsu Signed-off-by: Tony Luck --- include/asm-ia64/iosapic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/asm-ia64') diff --git a/include/asm-ia64/iosapic.h b/include/asm-ia64/iosapic.h index 421cb6b62a7c..09bdc3898df8 100644 --- a/include/asm-ia64/iosapic.h +++ b/include/asm-ia64/iosapic.h @@ -53,13 +53,13 @@ #define NR_IOSAPICS 256 -static inline unsigned int iosapic_read(char __iomem *iosapic, unsigned int reg) +static inline unsigned int __iosapic_read(char __iomem *iosapic, unsigned int reg) { writel(reg, iosapic + IOSAPIC_REG_SELECT); return readl(iosapic + IOSAPIC_WINDOW); } -static inline void iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val) +static inline void __iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val) { writel(reg, iosapic + IOSAPIC_REG_SELECT); writel(val, iosapic + IOSAPIC_WINDOW); -- cgit v1.2.3 From e1b30a392835e92581db09a4e8b4b2ad53a0c370 Mon Sep 17 00:00:00 2001 From: Yasuaki Ishimatsu Date: Tue, 17 Jul 2007 21:22:23 +0900 Subject: [IA64] Add mapping table between irq and vector Add mapping tables between irqs and vectors, and its management code. This is necessary for supporting multiple vector domain because 1:1 mapping between irq and vector will be changed to n:1. The irq == vector relationship between irqs and vectors is explicitly remained for percpu interrupts, platform interrupts, isa IRQs and vectors assigned using assign_irq_vector() because some programs might depend on it. And I should consider the following problem. When pci drivers enabled/disabled devices dynamically, its irq number is changed to the different one. Therefore, suspend/resume code may happen problem. To fix this problem, I bound gsi to irq. Signed-off-by: Kenji Kaneshige Signed-off-by: Yasuaki Ishimatsu Signed-off-by: Tony Luck --- arch/ia64/kernel/iosapic.c | 61 ++++++++------ arch/ia64/kernel/irq.c | 2 +- arch/ia64/kernel/irq_ia64.c | 201 ++++++++++++++++++++++++++++++++++++-------- arch/ia64/kernel/smpboot.c | 4 + include/asm-ia64/hw_irq.h | 15 +++- 5 files changed, 219 insertions(+), 64 deletions(-) (limited to 'include/asm-ia64') diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index cffb443a557c..cf27cfb4d165 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -117,6 +117,9 @@ static DEFINE_SPINLOCK(iosapic_lock); * These tables map IA-64 vectors to the IOSAPIC pin that generates this * vector. */ + +#define NO_REF_RTE 0 + static struct iosapic { char __iomem *addr; /* base address of IOSAPIC */ unsigned int gsi_base; /* GSI base */ @@ -204,7 +207,7 @@ inline int gsi_to_vector (unsigned int gsi) { int irq = __gsi_to_irq(gsi); - if (irq < 0) + if (check_irq_used(irq) < 0) return -1; return irq_to_vector(irq); } @@ -619,14 +622,18 @@ register_intr (unsigned int gsi, int irq, unsigned char delivery, iosapic_intr_info[irq].count++; iosapic_lists[index].rtes_inuse++; } - else if (irq_is_shared(irq)) { + else if (rte->refcnt == NO_REF_RTE) { struct iosapic_intr_info *info = &iosapic_intr_info[irq]; - if (info->trigger != trigger || info->polarity != polarity) { + if (info->count > 0 && + (info->trigger != trigger || info->polarity != polarity)){ printk (KERN_WARNING "%s: cannot override the interrupt\n", __FUNCTION__); return -EINVAL; } + rte->refcnt++; + iosapic_intr_info[irq].count++; + iosapic_lists[index].rtes_inuse++; } iosapic_intr_info[irq].polarity = polarity; @@ -756,12 +763,17 @@ iosapic_register_intr (unsigned int gsi, irq = __gsi_to_irq(gsi); if (irq > 0) { rte = find_rte(irq, gsi); - rte->refcnt++; - goto unlock_iosapic_lock; - } + if(iosapic_intr_info[irq].count == 0) { + assign_irq_vector(irq); + dynamic_irq_init(irq); + } else if (rte->refcnt != NO_REF_RTE) { + rte->refcnt++; + goto unlock_iosapic_lock; + } + } else + irq = create_irq(); /* If vector is running out, we try to find a sharable vector */ - irq = create_irq(); if (irq < 0) { irq = iosapic_find_sharable_irq(trigger, polarity); if (irq < 0) @@ -832,18 +844,14 @@ iosapic_unregister_intr (unsigned int gsi) if (--rte->refcnt > 0) goto out; - /* Remove the rte entry from the list */ idesc = irq_desc + irq; - spin_lock(&idesc->lock); - list_del(&rte->rte_list); - spin_unlock(&idesc->lock); + rte->refcnt = NO_REF_RTE; /* Mask the interrupt */ low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK; iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32); iosapic_intr_info[irq].count--; - iosapic_free_rte(rte); index = find_iosapic(gsi); iosapic_lists[index].rtes_inuse--; WARN_ON(iosapic_lists[index].rtes_inuse < 0); @@ -857,21 +865,20 @@ iosapic_unregister_intr (unsigned int gsi) (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), cpu_logical_id(dest), dest, irq_to_vector(irq)); - if (list_empty(&iosapic_intr_info[irq].rtes)) { - /* Sanity check */ - BUG_ON(iosapic_intr_info[irq].count); + if (iosapic_intr_info[irq].count == 0) { #ifdef CONFIG_SMP /* Clear affinity */ cpus_setall(idesc->affinity); #endif /* Clear the interrupt information */ - memset(&iosapic_intr_info[irq], 0, - sizeof(struct iosapic_intr_info)); + iosapic_intr_info[irq].dest = 0; + iosapic_intr_info[irq].dmode = 0; + iosapic_intr_info[irq].polarity = 0; + iosapic_intr_info[irq].trigger = 0; iosapic_intr_info[irq].low32 |= IOSAPIC_MASK; - INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); - /* Destroy IRQ */ - destroy_irq(irq); + /* Destroy and reserve IRQ */ + destroy_and_reserve_irq(irq); } out: spin_unlock_irqrestore(&iosapic_lock, flags); @@ -892,8 +899,8 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, switch (int_type) { case ACPI_INTERRUPT_PMI: - vector = iosapic_vector; - irq = vector; /* FIXME */ + irq = vector = iosapic_vector; + bind_irq_vector(irq, vector); /* * since PMI vector is alloc'd by FW(ACPI) not by kernel, * we need to make sure the vector is available @@ -909,8 +916,8 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, delivery = IOSAPIC_INIT; break; case ACPI_INTERRUPT_CPEI: - vector = IA64_CPE_VECTOR; - irq = vector; /* FIXME */ + irq = vector = IA64_CPE_VECTOR; + BUG_ON(bind_irq_vector(irq, vector)); delivery = IOSAPIC_LOWEST_PRIORITY; mask = 1; break; @@ -945,8 +952,8 @@ iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi, int vector, irq; unsigned int dest = cpu_physical_id(smp_processor_id()); - vector = isa_irq_to_vector(isa_irq); - irq = vector; /* FIXME */ + irq = vector = isa_irq_to_vector(isa_irq); + BUG_ON(bind_irq_vector(irq, vector)); register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", @@ -966,6 +973,8 @@ iosapic_system_init (int system_pcat_compat) iosapic_intr_info[irq].low32 = IOSAPIC_MASK; /* mark as unused */ INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); + + iosapic_intr_info[irq].count = 0; } pcat_compat = system_pcat_compat; diff --git a/arch/ia64/kernel/irq.c b/arch/ia64/kernel/irq.c index 407b45870489..cc3ee4ef37af 100644 --- a/arch/ia64/kernel/irq.c +++ b/arch/ia64/kernel/irq.c @@ -35,7 +35,7 @@ void ack_bad_irq(unsigned int irq) #ifdef CONFIG_IA64_GENERIC unsigned int __ia64_local_vector_to_irq (ia64_vector vec) { - return (unsigned int) vec; + return __get_cpu_var(vector_irq)[vec]; } #endif diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index bc47049f060f..072427c2c3f6 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -46,6 +46,12 @@ #define IRQ_DEBUG 0 +#define IRQ_VECTOR_UNASSIGNED (0) + +#define IRQ_UNUSED (0) +#define IRQ_USED (1) +#define IRQ_RSVD (2) + /* These can be overridden in platform_irq_init */ int ia64_first_device_vector = IA64_DEF_FIRST_DEVICE_VECTOR; int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR; @@ -64,46 +70,161 @@ __u8 isa_irq_to_vector_map[16] = { }; EXPORT_SYMBOL(isa_irq_to_vector_map); -static unsigned long ia64_vector_mask[BITS_TO_LONGS(IA64_MAX_DEVICE_VECTORS)]; +DEFINE_SPINLOCK(vector_lock); + +struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = { + [0 ... NR_IRQS - 1] = { .vector = IRQ_VECTOR_UNASSIGNED } +}; + +DEFINE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq) = { + [0 ... IA64_NUM_VECTORS - 1] = IA64_SPURIOUS_INT_VECTOR +}; + +static int irq_status[NR_IRQS] = { + [0 ... NR_IRQS -1] = IRQ_UNUSED +}; + +int check_irq_used(int irq) +{ + if (irq_status[irq] == IRQ_USED) + return 1; + + return -1; +} + +static void reserve_irq(unsigned int irq) +{ + unsigned long flags; + + spin_lock_irqsave(&vector_lock, flags); + irq_status[irq] = IRQ_RSVD; + spin_unlock_irqrestore(&vector_lock, flags); +} + +static inline int find_unassigned_irq(void) +{ + int irq; + + for (irq = IA64_FIRST_DEVICE_VECTOR; irq < NR_IRQS; irq++) + if (irq_status[irq] == IRQ_UNUSED) + return irq; + return -ENOSPC; +} + +static inline int find_unassigned_vector(void) +{ + int vector; + + for (vector = IA64_FIRST_DEVICE_VECTOR; + vector <= IA64_LAST_DEVICE_VECTOR; vector++) + if (__get_cpu_var(vector_irq[vector]) == IA64_SPURIOUS_INT_VECTOR) + return vector; + return -ENOSPC; +} + +static int __bind_irq_vector(int irq, int vector) +{ + int cpu; + + if (irq_to_vector(irq) == vector) + return 0; + if (irq_to_vector(irq) != IRQ_VECTOR_UNASSIGNED) + return -EBUSY; + for_each_online_cpu(cpu) + per_cpu(vector_irq, cpu)[vector] = irq; + irq_cfg[irq].vector = vector; + irq_status[irq] = IRQ_USED; + return 0; +} + +int bind_irq_vector(int irq, int vector) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&vector_lock, flags); + ret = __bind_irq_vector(irq, vector); + spin_unlock_irqrestore(&vector_lock, flags); + return ret; +} + +static void clear_irq_vector(int irq) +{ + unsigned long flags; + int vector, cpu; + + spin_lock_irqsave(&vector_lock, flags); + BUG_ON((unsigned)irq >= NR_IRQS); + BUG_ON(irq_cfg[irq].vector == IRQ_VECTOR_UNASSIGNED); + vector = irq_cfg[irq].vector; + for_each_online_cpu(cpu) + per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR; + irq_cfg[irq].vector = IRQ_VECTOR_UNASSIGNED; + irq_status[irq] = IRQ_UNUSED; + spin_unlock_irqrestore(&vector_lock, flags); +} int assign_irq_vector (int irq) { - int pos, vector; - again: - pos = find_first_zero_bit(ia64_vector_mask, IA64_NUM_DEVICE_VECTORS); - vector = IA64_FIRST_DEVICE_VECTOR + pos; - if (vector > IA64_LAST_DEVICE_VECTOR) - return -ENOSPC; - if (test_and_set_bit(pos, ia64_vector_mask)) - goto again; + unsigned long flags; + int vector = -ENOSPC; + + if (irq < 0) { + goto out; + } + spin_lock_irqsave(&vector_lock, flags); + vector = find_unassigned_vector(); + if (vector < 0) + goto out; + BUG_ON(__bind_irq_vector(irq, vector)); + spin_unlock_irqrestore(&vector_lock, flags); + out: return vector; } void free_irq_vector (int vector) { - int pos; - - if (vector < IA64_FIRST_DEVICE_VECTOR || vector > IA64_LAST_DEVICE_VECTOR) + if (vector < IA64_FIRST_DEVICE_VECTOR || + vector > IA64_LAST_DEVICE_VECTOR) return; - - pos = vector - IA64_FIRST_DEVICE_VECTOR; - if (!test_and_clear_bit(pos, ia64_vector_mask)) - printk(KERN_WARNING "%s: double free!\n", __FUNCTION__); + clear_irq_vector(vector); } int reserve_irq_vector (int vector) { - int pos; - if (vector < IA64_FIRST_DEVICE_VECTOR || vector > IA64_LAST_DEVICE_VECTOR) return -EINVAL; + return !!bind_irq_vector(vector, vector); +} - pos = vector - IA64_FIRST_DEVICE_VECTOR; - return test_and_set_bit(pos, ia64_vector_mask); +/* + * Initialize vector_irq on a new cpu. This function must be called + * with vector_lock held. + */ +void __setup_vector_irq(int cpu) +{ + int irq, vector; + + /* Clear vector_irq */ + for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) + per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR; + /* Mark the inuse vectors */ + for (irq = 0; irq < NR_IRQS; ++irq) { + if ((vector = irq_to_vector(irq)) != IRQ_VECTOR_UNASSIGNED) + per_cpu(vector_irq, cpu)[vector] = irq; + } +} + +void destroy_and_reserve_irq(unsigned int irq) +{ + dynamic_irq_cleanup(irq); + + clear_irq_vector(irq); + reserve_irq(irq); } /* @@ -111,18 +232,29 @@ reserve_irq_vector (int vector) */ int create_irq(void) { - int vector = assign_irq_vector(AUTO_ASSIGN); - - if (vector >= 0) - dynamic_irq_init(vector); - - return vector; + unsigned long flags; + int irq, vector; + + irq = -ENOSPC; + spin_lock_irqsave(&vector_lock, flags); + vector = find_unassigned_vector(); + if (vector < 0) + goto out; + irq = find_unassigned_irq(); + if (irq < 0) + goto out; + BUG_ON(__bind_irq_vector(irq, vector)); + out: + spin_unlock_irqrestore(&vector_lock, flags); + if (irq >= 0) + dynamic_irq_init(irq); + return irq; } void destroy_irq(unsigned int irq) { dynamic_irq_cleanup(irq); - free_irq_vector(irq); + clear_irq_vector(irq); } #ifdef CONFIG_SMP @@ -301,14 +433,13 @@ register_percpu_irq (ia64_vector vec, struct irqaction *action) irq_desc_t *desc; unsigned int irq; - for (irq = 0; irq < NR_IRQS; ++irq) - if (irq_to_vector(irq) == vec) { - desc = irq_desc + irq; - desc->status |= IRQ_PER_CPU; - desc->chip = &irq_type_ia64_lsapic; - if (action) - setup_irq(irq, action); - } + irq = vec; + BUG_ON(bind_irq_vector(irq, vec)); + desc = irq_desc + irq; + desc->status |= IRQ_PER_CPU; + desc->chip = &irq_type_ia64_lsapic; + if (action) + setup_irq(irq, action); } void __init diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 3c9d8e6089cf..9f5c90b594b9 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -395,9 +395,13 @@ smp_callin (void) fix_b0_for_bsp(); lock_ipi_calllock(); + spin_lock(&vector_lock); + /* Setup the per cpu irq handling data structures */ + __setup_vector_irq(cpuid); cpu_set(cpuid, cpu_online_map); unlock_ipi_calllock(); per_cpu(cpu_state, cpuid) = CPU_ONLINE; + spin_unlock(&vector_lock); smp_setup_percpu_timer(); diff --git a/include/asm-ia64/hw_irq.h b/include/asm-ia64/hw_irq.h index c054d7a9aaa7..4eff7ff2da82 100644 --- a/include/asm-ia64/hw_irq.h +++ b/include/asm-ia64/hw_irq.h @@ -90,13 +90,24 @@ enum { extern __u8 isa_irq_to_vector_map[16]; #define isa_irq_to_vector(x) isa_irq_to_vector_map[(x)] +struct irq_cfg { + ia64_vector vector; +}; +extern spinlock_t vector_lock; +extern struct irq_cfg irq_cfg[NR_IRQS]; +DECLARE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq); + extern struct hw_interrupt_type irq_type_ia64_lsapic; /* CPU-internal interrupt controller */ +extern int bind_irq_vector(int irq, int vector); extern int assign_irq_vector (int irq); /* allocate a free vector */ extern void free_irq_vector (int vector); extern int reserve_irq_vector (int vector); +extern void __setup_vector_irq(int cpu); extern void ia64_send_ipi (int cpu, int vector, int delivery_mode, int redirect); extern void register_percpu_irq (ia64_vector vec, struct irqaction *action); +extern int check_irq_used (int irq); +extern void destroy_and_reserve_irq (unsigned int irq); static inline void ia64_resend_irq(unsigned int vector) { @@ -113,7 +124,7 @@ extern irq_desc_t irq_desc[NR_IRQS]; static inline unsigned int __ia64_local_vector_to_irq (ia64_vector vec) { - return (unsigned int) vec; + return __get_cpu_var(vector_irq)[vec]; } #endif @@ -131,7 +142,7 @@ __ia64_local_vector_to_irq (ia64_vector vec) static inline ia64_vector irq_to_vector (int irq) { - return (ia64_vector) irq; + return irq_cfg[irq].vector; } /* -- cgit v1.2.3 From 4994be1b3fe9120c88022ff5c0c33f6312b17adb Mon Sep 17 00:00:00 2001 From: Yasuaki Ishimatsu Date: Tue, 17 Jul 2007 21:22:33 +0900 Subject: [IA64] Add support for vector domain Add fundamental support for multiple vector domain. There still exists only one vector domain even with this patch. IRQ migration across domain is not supported yet by this patch. Signed-off-by: Kenji Kaneshige Signed-off-by: Yasuaki Ishimatsu Signed-off-by: Tony Luck --- arch/ia64/kernel/iosapic.c | 13 +++-- arch/ia64/kernel/irq_ia64.c | 120 ++++++++++++++++++++++++++++++++------------ arch/ia64/kernel/msi_ia64.c | 9 +++- include/asm-ia64/hw_irq.h | 4 +- include/asm-ia64/irq.h | 9 +++- 5 files changed, 113 insertions(+), 42 deletions(-) (limited to 'include/asm-ia64') diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index cf27cfb4d165..e647254c2707 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -354,6 +354,8 @@ iosapic_set_affinity (unsigned int irq, cpumask_t mask) irq &= (~IA64_IRQ_REDIRECTED); + /* IRQ migration across domain is not supported yet */ + cpus_and(mask, mask, irq_to_domain(irq)); if (cpus_empty(mask)) return; @@ -663,6 +665,7 @@ get_target_cpu (unsigned int gsi, int irq) #ifdef CONFIG_SMP static int cpu = -1; extern int cpe_vector; + cpumask_t domain = irq_to_domain(irq); /* * In case of vector shared by multiple RTEs, all RTEs that @@ -701,7 +704,7 @@ get_target_cpu (unsigned int gsi, int irq) goto skip_numa_setup; cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node); - + cpus_and(cpu_mask, cpu_mask, domain); for_each_cpu_mask(numa_cpu, cpu_mask) { if (!cpu_online(numa_cpu)) cpu_clear(numa_cpu, cpu_mask); @@ -731,7 +734,7 @@ skip_numa_setup: do { if (++cpu >= NR_CPUS) cpu = 0; - } while (!cpu_online(cpu)); + } while (!cpu_online(cpu) || !cpu_isset(cpu, domain)); return cpu_physical_id(cpu); #else /* CONFIG_SMP */ @@ -900,7 +903,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, switch (int_type) { case ACPI_INTERRUPT_PMI: irq = vector = iosapic_vector; - bind_irq_vector(irq, vector); + bind_irq_vector(irq, vector, CPU_MASK_ALL); /* * since PMI vector is alloc'd by FW(ACPI) not by kernel, * we need to make sure the vector is available @@ -917,7 +920,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi, break; case ACPI_INTERRUPT_CPEI: irq = vector = IA64_CPE_VECTOR; - BUG_ON(bind_irq_vector(irq, vector)); + BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); delivery = IOSAPIC_LOWEST_PRIORITY; mask = 1; break; @@ -953,7 +956,7 @@ iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi, unsigned int dest = cpu_physical_id(smp_processor_id()); irq = vector = isa_irq_to_vector(isa_irq); - BUG_ON(bind_irq_vector(irq, vector)); + BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index 072427c2c3f6..a3667631ed80 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -60,6 +60,8 @@ int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR; void __iomem *ipi_base_addr = ((void __iomem *) (__IA64_UNCACHED_OFFSET | IA64_IPI_DEFAULT_BASE_ADDR)); +static cpumask_t vector_allocation_domain(int cpu); + /* * Legacy IRQ to IA-64 vector translation table. */ @@ -73,13 +75,20 @@ EXPORT_SYMBOL(isa_irq_to_vector_map); DEFINE_SPINLOCK(vector_lock); struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = { - [0 ... NR_IRQS - 1] = { .vector = IRQ_VECTOR_UNASSIGNED } + [0 ... NR_IRQS - 1] = { + .vector = IRQ_VECTOR_UNASSIGNED, + .domain = CPU_MASK_NONE + } }; DEFINE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq) = { [0 ... IA64_NUM_VECTORS - 1] = IA64_SPURIOUS_INT_VECTOR }; +static cpumask_t vector_table[IA64_MAX_DEVICE_VECTORS] = { + [0 ... IA64_MAX_DEVICE_VECTORS - 1] = CPU_MASK_NONE +}; + static int irq_status[NR_IRQS] = { [0 ... NR_IRQS -1] = IRQ_UNUSED }; @@ -111,39 +120,54 @@ static inline int find_unassigned_irq(void) return -ENOSPC; } -static inline int find_unassigned_vector(void) +static inline int find_unassigned_vector(cpumask_t domain) { - int vector; + cpumask_t mask; + int pos; + + cpus_and(mask, domain, cpu_online_map); + if (cpus_empty(mask)) + return -EINVAL; - for (vector = IA64_FIRST_DEVICE_VECTOR; - vector <= IA64_LAST_DEVICE_VECTOR; vector++) - if (__get_cpu_var(vector_irq[vector]) == IA64_SPURIOUS_INT_VECTOR) - return vector; + for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) { + cpus_and(mask, domain, vector_table[pos]); + if (!cpus_empty(mask)) + continue; + return IA64_FIRST_DEVICE_VECTOR + pos; + } return -ENOSPC; } -static int __bind_irq_vector(int irq, int vector) +static int __bind_irq_vector(int irq, int vector, cpumask_t domain) { - int cpu; + cpumask_t mask; + int cpu, pos; + struct irq_cfg *cfg = &irq_cfg[irq]; - if (irq_to_vector(irq) == vector) + cpus_and(mask, domain, cpu_online_map); + if (cpus_empty(mask)) + return -EINVAL; + if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain)) return 0; - if (irq_to_vector(irq) != IRQ_VECTOR_UNASSIGNED) + if (cfg->vector != IRQ_VECTOR_UNASSIGNED) return -EBUSY; - for_each_online_cpu(cpu) + for_each_cpu_mask(cpu, mask) per_cpu(vector_irq, cpu)[vector] = irq; - irq_cfg[irq].vector = vector; + cfg->vector = vector; + cfg->domain = domain; irq_status[irq] = IRQ_USED; + pos = vector - IA64_FIRST_DEVICE_VECTOR; + cpus_or(vector_table[pos], vector_table[pos], domain); return 0; } -int bind_irq_vector(int irq, int vector) +int bind_irq_vector(int irq, int vector, cpumask_t domain) { unsigned long flags; int ret; spin_lock_irqsave(&vector_lock, flags); - ret = __bind_irq_vector(irq, vector); + ret = __bind_irq_vector(irq, vector, domain); spin_unlock_irqrestore(&vector_lock, flags); return ret; } @@ -151,16 +175,24 @@ int bind_irq_vector(int irq, int vector) static void clear_irq_vector(int irq) { unsigned long flags; - int vector, cpu; + int vector, cpu, pos; + cpumask_t mask; + cpumask_t domain; + struct irq_cfg *cfg = &irq_cfg[irq]; spin_lock_irqsave(&vector_lock, flags); BUG_ON((unsigned)irq >= NR_IRQS); - BUG_ON(irq_cfg[irq].vector == IRQ_VECTOR_UNASSIGNED); - vector = irq_cfg[irq].vector; - for_each_online_cpu(cpu) + BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); + vector = cfg->vector; + domain = cfg->domain; + cpus_and(mask, cfg->domain, cpu_online_map); + for_each_cpu_mask(cpu, mask) per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR; - irq_cfg[irq].vector = IRQ_VECTOR_UNASSIGNED; + cfg->vector = IRQ_VECTOR_UNASSIGNED; + cfg->domain = CPU_MASK_NONE; irq_status[irq] = IRQ_UNUSED; + pos = vector - IA64_FIRST_DEVICE_VECTOR; + cpus_andnot(vector_table[pos], vector_table[pos], domain); spin_unlock_irqrestore(&vector_lock, flags); } @@ -168,18 +200,26 @@ int assign_irq_vector (int irq) { unsigned long flags; - int vector = -ENOSPC; + int vector, cpu; + cpumask_t domain; + + vector = -ENOSPC; + spin_lock_irqsave(&vector_lock, flags); if (irq < 0) { goto out; } - spin_lock_irqsave(&vector_lock, flags); - vector = find_unassigned_vector(); + for_each_online_cpu(cpu) { + domain = vector_allocation_domain(cpu); + vector = find_unassigned_vector(domain); + if (vector >= 0) + break; + } if (vector < 0) goto out; - BUG_ON(__bind_irq_vector(irq, vector)); - spin_unlock_irqrestore(&vector_lock, flags); + BUG_ON(__bind_irq_vector(irq, vector, domain)); out: + spin_unlock_irqrestore(&vector_lock, flags); return vector; } @@ -198,7 +238,7 @@ reserve_irq_vector (int vector) if (vector < IA64_FIRST_DEVICE_VECTOR || vector > IA64_LAST_DEVICE_VECTOR) return -EINVAL; - return !!bind_irq_vector(vector, vector); + return !!bind_irq_vector(vector, vector, CPU_MASK_ALL); } /* @@ -214,11 +254,19 @@ void __setup_vector_irq(int cpu) per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR; /* Mark the inuse vectors */ for (irq = 0; irq < NR_IRQS; ++irq) { - if ((vector = irq_to_vector(irq)) != IRQ_VECTOR_UNASSIGNED) - per_cpu(vector_irq, cpu)[vector] = irq; + if (!cpu_isset(cpu, irq_cfg[irq].domain)) + continue; + vector = irq_to_vector(irq); + per_cpu(vector_irq, cpu)[vector] = irq; } } +static cpumask_t vector_allocation_domain(int cpu) +{ + return CPU_MASK_ALL; +} + + void destroy_and_reserve_irq(unsigned int irq) { dynamic_irq_cleanup(irq); @@ -233,17 +281,23 @@ void destroy_and_reserve_irq(unsigned int irq) int create_irq(void) { unsigned long flags; - int irq, vector; + int irq, vector, cpu; + cpumask_t domain; - irq = -ENOSPC; + irq = vector = -ENOSPC; spin_lock_irqsave(&vector_lock, flags); - vector = find_unassigned_vector(); + for_each_online_cpu(cpu) { + domain = vector_allocation_domain(cpu); + vector = find_unassigned_vector(domain); + if (vector >= 0) + break; + } if (vector < 0) goto out; irq = find_unassigned_irq(); if (irq < 0) goto out; - BUG_ON(__bind_irq_vector(irq, vector)); + BUG_ON(__bind_irq_vector(irq, vector, domain)); out: spin_unlock_irqrestore(&vector_lock, flags); if (irq >= 0) @@ -434,7 +488,7 @@ register_percpu_irq (ia64_vector vec, struct irqaction *action) unsigned int irq; irq = vec; - BUG_ON(bind_irq_vector(irq, vec)); + BUG_ON(bind_irq_vector(irq, vec, CPU_MASK_ALL)); desc = irq_desc + irq; desc->status |= IRQ_PER_CPU; desc->chip = &irq_type_ia64_lsapic; diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index c81080df70df..1d22670cc88b 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -52,6 +52,11 @@ static void ia64_set_msi_irq_affinity(unsigned int irq, cpumask_t cpu_mask) struct msi_msg msg; u32 addr; + /* IRQ migration across domain is not supported yet */ + cpus_and(cpu_mask, cpu_mask, irq_to_domain(irq)); + if (cpus_empty(cpu_mask)) + return; + read_msi_msg(irq, &msg); addr = msg.address_lo; @@ -69,13 +74,15 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) struct msi_msg msg; unsigned long dest_phys_id; int irq, vector; + cpumask_t mask; irq = create_irq(); if (irq < 0) return irq; set_irq_msi(irq, desc); - dest_phys_id = cpu_physical_id(first_cpu(cpu_online_map)); + cpus_and(mask, irq_to_domain(irq), cpu_online_map); + dest_phys_id = cpu_physical_id(first_cpu(mask)); vector = irq_to_vector(irq); msg.address_hi = 0; diff --git a/include/asm-ia64/hw_irq.h b/include/asm-ia64/hw_irq.h index 4eff7ff2da82..97ea3900d74a 100644 --- a/include/asm-ia64/hw_irq.h +++ b/include/asm-ia64/hw_irq.h @@ -92,14 +92,16 @@ extern __u8 isa_irq_to_vector_map[16]; struct irq_cfg { ia64_vector vector; + cpumask_t domain; }; extern spinlock_t vector_lock; extern struct irq_cfg irq_cfg[NR_IRQS]; +#define irq_to_domain(x) irq_cfg[(x)].domain DECLARE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq); extern struct hw_interrupt_type irq_type_ia64_lsapic; /* CPU-internal interrupt controller */ -extern int bind_irq_vector(int irq, int vector); +extern int bind_irq_vector(int irq, int vector, cpumask_t domain); extern int assign_irq_vector (int irq); /* allocate a free vector */ extern void free_irq_vector (int vector); extern int reserve_irq_vector (int vector); diff --git a/include/asm-ia64/irq.h b/include/asm-ia64/irq.h index 67221615e317..35b360b82e43 100644 --- a/include/asm-ia64/irq.h +++ b/include/asm-ia64/irq.h @@ -14,8 +14,13 @@ #include #include -#define NR_IRQS 256 -#define NR_IRQ_VECTORS NR_IRQS +#define NR_VECTORS 256 + +#if (NR_VECTORS + 32 * NR_CPUS) < 1024 +#define NR_IRQS (NR_VECTORS + 32 * NR_CPUS) +#else +#define NR_IRQS 1024 +#endif static __inline__ int irq_canonicalize (int irq) -- cgit v1.2.3 From cd378f18cf73d92bf0b6e1e6b5759b5dd729a9f2 Mon Sep 17 00:00:00 2001 From: Yasuaki Ishimatsu Date: Tue, 17 Jul 2007 21:22:48 +0900 Subject: [IA64] Support irq migration across domain Add support for IRQ migration across vector domain. Signed-off-by: Kenji Kaneshige Signed-off-by: Yasuaki Ishimatsu Signed-off-by: Tony Luck --- arch/ia64/kernel/iosapic.c | 20 +++++++++++++++++--- arch/ia64/kernel/irq_ia64.c | 42 +++++++++++++++++++++++++++++++++++++++--- arch/ia64/kernel/msi_ia64.c | 20 ++++++++++++++------ include/asm-ia64/hw_irq.h | 1 + include/asm-ia64/iosapic.h | 2 ++ 5 files changed, 73 insertions(+), 12 deletions(-) (limited to 'include/asm-ia64') diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index e647254c2707..c101c8bff27b 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -354,11 +354,13 @@ iosapic_set_affinity (unsigned int irq, cpumask_t mask) irq &= (~IA64_IRQ_REDIRECTED); - /* IRQ migration across domain is not supported yet */ - cpus_and(mask, mask, irq_to_domain(irq)); + cpus_and(mask, mask, cpu_online_map); if (cpus_empty(mask)) return; + if (reassign_irq_vector(irq, first_cpu(mask))) + return; + dest = cpu_physical_id(first_cpu(mask)); if (list_empty(&iosapic_intr_info[irq].rtes)) @@ -376,6 +378,8 @@ iosapic_set_affinity (unsigned int irq, cpumask_t mask) else /* change delivery mode to fixed */ low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); + low32 &= IOSAPIC_VECTOR_MASK; + low32 |= irq_to_vector(irq); iosapic_intr_info[irq].low32 = low32; iosapic_intr_info[irq].dest = dest; @@ -404,10 +408,20 @@ iosapic_end_level_irq (unsigned int irq) { ia64_vector vec = irq_to_vector(irq); struct iosapic_rte_info *rte; + int do_unmask_irq = 0; + + if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) { + do_unmask_irq = 1; + mask_irq(irq); + } - move_native_irq(irq); list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) iosapic_eoi(rte->iosapic->addr, vec); + + if (unlikely(do_unmask_irq)) { + move_masked_irq(irq); + unmask_irq(irq); + } } #define iosapic_shutdown_level_irq mask_irq diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index a3667631ed80..22806b94025a 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -172,15 +172,13 @@ int bind_irq_vector(int irq, int vector, cpumask_t domain) return ret; } -static void clear_irq_vector(int irq) +static void __clear_irq_vector(int irq) { - unsigned long flags; int vector, cpu, pos; cpumask_t mask; cpumask_t domain; struct irq_cfg *cfg = &irq_cfg[irq]; - spin_lock_irqsave(&vector_lock, flags); BUG_ON((unsigned)irq >= NR_IRQS); BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); vector = cfg->vector; @@ -193,6 +191,14 @@ static void clear_irq_vector(int irq) irq_status[irq] = IRQ_UNUSED; pos = vector - IA64_FIRST_DEVICE_VECTOR; cpus_andnot(vector_table[pos], vector_table[pos], domain); +} + +static void clear_irq_vector(int irq) +{ + unsigned long flags; + + spin_lock_irqsave(&vector_lock, flags); + __clear_irq_vector(irq); spin_unlock_irqrestore(&vector_lock, flags); } @@ -275,6 +281,36 @@ void destroy_and_reserve_irq(unsigned int irq) reserve_irq(irq); } +static int __reassign_irq_vector(int irq, int cpu) +{ + struct irq_cfg *cfg = &irq_cfg[irq]; + int vector; + cpumask_t domain; + + if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu)) + return -EINVAL; + if (cpu_isset(cpu, cfg->domain)) + return 0; + domain = vector_allocation_domain(cpu); + vector = find_unassigned_vector(domain); + if (vector < 0) + return -ENOSPC; + __clear_irq_vector(irq); + BUG_ON(__bind_irq_vector(irq, vector, domain)); + return 0; +} + +int reassign_irq_vector(int irq, int cpu) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&vector_lock, flags); + ret = __reassign_irq_vector(irq, cpu); + spin_unlock_irqrestore(&vector_lock, flags); + return ret; +} + /* * Dynamic irq allocate and deallocation for MSI */ diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index 1d22670cc88b..2fdbd5c3f213 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -13,6 +13,7 @@ #define MSI_DATA_VECTOR_SHIFT 0 #define MSI_DATA_VECTOR(v) (((u8)v) << MSI_DATA_VECTOR_SHIFT) +#define MSI_DATA_VECTOR_MASK 0xffffff00 #define MSI_DATA_DELIVERY_SHIFT 8 #define MSI_DATA_DELIVERY_FIXED (0 << MSI_DATA_DELIVERY_SHIFT) @@ -50,22 +51,29 @@ static struct irq_chip ia64_msi_chip; static void ia64_set_msi_irq_affinity(unsigned int irq, cpumask_t cpu_mask) { struct msi_msg msg; - u32 addr; + u32 addr, data; + int cpu = first_cpu(cpu_mask); - /* IRQ migration across domain is not supported yet */ - cpus_and(cpu_mask, cpu_mask, irq_to_domain(irq)); - if (cpus_empty(cpu_mask)) + if (!cpu_online(cpu)) + return; + + if (reassign_irq_vector(irq, cpu)) return; read_msi_msg(irq, &msg); addr = msg.address_lo; addr &= MSI_ADDR_DESTID_MASK; - addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(first_cpu(cpu_mask))); + addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(cpu)); msg.address_lo = addr; + data = msg.data; + data &= MSI_DATA_VECTOR_MASK; + data |= MSI_DATA_VECTOR(irq_to_vector(irq)); + msg.data = data; + write_msi_msg(irq, &msg); - irq_desc[irq].affinity = cpu_mask; + irq_desc[irq].affinity = cpumask_of_cpu(cpu); } #endif /* CONFIG_SMP */ diff --git a/include/asm-ia64/hw_irq.h b/include/asm-ia64/hw_irq.h index 97ea3900d74a..efa1b8f7251d 100644 --- a/include/asm-ia64/hw_irq.h +++ b/include/asm-ia64/hw_irq.h @@ -106,6 +106,7 @@ extern int assign_irq_vector (int irq); /* allocate a free vector */ extern void free_irq_vector (int vector); extern int reserve_irq_vector (int vector); extern void __setup_vector_irq(int cpu); +extern int reassign_irq_vector(int irq, int cpu); extern void ia64_send_ipi (int cpu, int vector, int delivery_mode, int redirect); extern void register_percpu_irq (ia64_vector vec, struct irqaction *action); extern int check_irq_used (int irq); diff --git a/include/asm-ia64/iosapic.h b/include/asm-ia64/iosapic.h index 09bdc3898df8..b8f712859140 100644 --- a/include/asm-ia64/iosapic.h +++ b/include/asm-ia64/iosapic.h @@ -47,6 +47,8 @@ #define IOSAPIC_MASK_SHIFT 16 #define IOSAPIC_MASK (1< Date: Mon, 16 Jul 2007 15:33:40 +1000 Subject: [IA64] fallocate system call sys_fallocate for ia64. This uses an empty slot #1303 erroneously marked as reserved for move_pages (which had already been allocated as syscall #1276) Signed-Off-By: Dave Chinner Signed-off-by: Tony Luck --- arch/ia64/kernel/entry.S | 2 +- include/asm-ia64/unistd.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/asm-ia64') diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index 95f517515235..c36f43c94600 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S @@ -1581,7 +1581,7 @@ sys_call_table: data8 sys_sync_file_range // 1300 data8 sys_tee data8 sys_vmsplice - data8 sys_ni_syscall // reserved for move_pages + data8 sys_fallocate data8 sys_getcpu data8 sys_epoll_pwait // 1305 data8 sys_utimensat diff --git a/include/asm-ia64/unistd.h b/include/asm-ia64/unistd.h index 441c9e001776..315f8de950a2 100644 --- a/include/asm-ia64/unistd.h +++ b/include/asm-ia64/unistd.h @@ -292,7 +292,7 @@ #define __NR_sync_file_range 1300 #define __NR_tee 1301 #define __NR_vmsplice 1302 -/* 1303 reserved for move_pages */ +#define __NR_fallocate 1303 #define __NR_getcpu 1304 #define __NR_epoll_pwait 1305 #define __NR_utimensat 1306 -- cgit v1.2.3 From bd807f9c5b90c236c3d173a5815d620bc1b1316d Mon Sep 17 00:00:00 2001 From: "Robert P. J. Day" Date: Wed, 18 Jul 2007 07:33:08 -0400 Subject: [IA64] Prevent people from directly including . Signed-off-by: Robert P. J. Day Signed-off-by: Tony Luck --- include/asm-ia64/rwsem.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/asm-ia64') diff --git a/include/asm-ia64/rwsem.h b/include/asm-ia64/rwsem.h index 2d1640cc240a..8aba06a7b038 100644 --- a/include/asm-ia64/rwsem.h +++ b/include/asm-ia64/rwsem.h @@ -21,6 +21,10 @@ #ifndef _ASM_IA64_RWSEM_H #define _ASM_IA64_RWSEM_H +#ifndef _LINUX_RWSEM_H +#error "Please don't include directly, use instead." +#endif + #include #include -- cgit v1.2.3