From dacd45f4e793e46e8299c9a580e400866ffe0770 Mon Sep 17 00:00:00 2001 From: Konrad Rzeszutek Wilk Date: Mon, 22 Oct 2012 11:35:16 -0400 Subject: xen/smp: Move the common CPU init code a bit to prep for PVH patch. The PV and PVH code CPU init code share some functionality. The PVH code ("xen/pvh: Extend vcpu_guest_context, p2m, event, and XenBus") sets some of these up, but not all. To make it easier to read, this patch removes the PV specific out of the generic way. No functional change - just code movement. Acked-by: Stefano Stabellini [v2: Fixed compile errors noticed by Fengguang Wu build system] Signed-off-by: Konrad Rzeszutek Wilk --- arch/x86/xen/smp.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) (limited to 'arch') diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 34bc4cee8887..09ea61d2e02f 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -300,8 +300,6 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) gdt = get_cpu_gdt_table(cpu); ctxt->flags = VGCF_IN_KERNEL; - ctxt->user_regs.ds = __USER_DS; - ctxt->user_regs.es = __USER_DS; ctxt->user_regs.ss = __KERNEL_DS; #ifdef CONFIG_X86_32 ctxt->user_regs.fs = __KERNEL_PERCPU; @@ -310,35 +308,41 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) ctxt->gs_base_kernel = per_cpu_offset(cpu); #endif ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle; - ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */ memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt)); - xen_copy_trap_info(ctxt->trap_ctxt); + { + ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */ + ctxt->user_regs.ds = __USER_DS; + ctxt->user_regs.es = __USER_DS; - ctxt->ldt_ents = 0; + xen_copy_trap_info(ctxt->trap_ctxt); - BUG_ON((unsigned long)gdt & ~PAGE_MASK); + ctxt->ldt_ents = 0; - gdt_mfn = arbitrary_virt_to_mfn(gdt); - make_lowmem_page_readonly(gdt); - make_lowmem_page_readonly(mfn_to_virt(gdt_mfn)); + BUG_ON((unsigned long)gdt & ~PAGE_MASK); - ctxt->gdt_frames[0] = gdt_mfn; - ctxt->gdt_ents = GDT_ENTRIES; + gdt_mfn = arbitrary_virt_to_mfn(gdt); + make_lowmem_page_readonly(gdt); + make_lowmem_page_readonly(mfn_to_virt(gdt_mfn)); - ctxt->user_regs.cs = __KERNEL_CS; - ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); + ctxt->gdt_frames[0] = gdt_mfn; + ctxt->gdt_ents = GDT_ENTRIES; - ctxt->kernel_ss = __KERNEL_DS; - ctxt->kernel_sp = idle->thread.sp0; + ctxt->kernel_ss = __KERNEL_DS; + ctxt->kernel_sp = idle->thread.sp0; #ifdef CONFIG_X86_32 - ctxt->event_callback_cs = __KERNEL_CS; - ctxt->failsafe_callback_cs = __KERNEL_CS; + ctxt->event_callback_cs = __KERNEL_CS; + ctxt->failsafe_callback_cs = __KERNEL_CS; #endif - ctxt->event_callback_eip = (unsigned long)xen_hypervisor_callback; - ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback; + ctxt->event_callback_eip = + (unsigned long)xen_hypervisor_callback; + ctxt->failsafe_callback_eip = + (unsigned long)xen_failsafe_callback; + } + ctxt->user_regs.cs = __KERNEL_CS; + ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir); ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir)); -- cgit v1.2.3 From 07d0c943663f82d9682856c0a7db7145a6c911d6 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 19 Feb 2013 22:00:58 -0500 Subject: xen: implement updated XENMEM_add_to_physmap_range ABI Allows for more fine grained error reporting. Only used by PVH and ARM both of which are marked EXPERIMENTAL precisely because the ABI is not yet stable Signed-off-by: Ian Campbell [v1: Rebased without PVH patches] Signed-off-by: Konrad Rzeszutek Wilk --- arch/arm/xen/enlighten.c | 8 +++++--- include/xen/interface/memory.h | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 7a32976fa2a3..8dc0605a9ce9 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -59,14 +59,16 @@ static int map_foreign_page(unsigned long lpfn, unsigned long fgmfn, }; xen_ulong_t idx = fgmfn; xen_pfn_t gpfn = lpfn; + int err = 0; set_xen_guest_handle(xatp.idxs, &idx); set_xen_guest_handle(xatp.gpfns, &gpfn); + set_xen_guest_handle(xatp.errs, &err); rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp); - if (rc) { - pr_warn("Failed to map pfn to mfn rc:%d pfn:%lx mfn:%lx\n", - rc, lpfn, fgmfn); + if (rc || err) { + pr_warn("Failed to map pfn to mfn rc:%d:%d pfn:%lx mfn:%lx\n", + rc, err, lpfn, fgmfn); return 1; } return 0; diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h index b40a4315cb8b..2ecfe4f700d9 100644 --- a/include/xen/interface/memory.h +++ b/include/xen/interface/memory.h @@ -190,6 +190,7 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_add_to_physmap); #define XENMEM_add_to_physmap_range 23 struct xen_add_to_physmap_range { + /* IN */ /* Which domain to change the mapping for. */ domid_t domid; uint16_t space; /* => enum phys_map_space */ @@ -203,6 +204,11 @@ struct xen_add_to_physmap_range { /* GPFN in domid where the source mapping page should appear. */ GUEST_HANDLE(xen_pfn_t) gpfns; + + /* OUT */ + + /* Per index error code. */ + GUEST_HANDLE(int) errs; }; DEFINE_GUEST_HANDLE_STRUCT(xen_add_to_physmap_range); -- cgit v1.2.3 From 3216dceb31c08be08ea98814a9ca5775fa680389 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Tue, 19 Feb 2013 13:59:19 +0000 Subject: xen: introduce xen_remap, use it instead of ioremap ioremap can't be used to map ring pages on ARM because it uses device memory caching attributes (MT_DEVICE*). Introduce a Xen specific abstraction to map ring pages, called xen_remap, that is defined as ioremap on x86 (no behavioral changes). On ARM it explicitly calls __arm_ioremap with the right caching attributes: MT_MEMORY. Signed-off-by: Stefano Stabellini Signed-off-by: Konrad Rzeszutek Wilk --- arch/arm/include/asm/xen/page.h | 4 ++++ arch/x86/include/asm/xen/page.h | 2 ++ drivers/tty/hvc/hvc_xen.c | 2 +- drivers/xen/grant-table.c | 2 +- drivers/xen/xenbus/xenbus_probe.c | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index c6b9096cef95..30cdacb675af 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h @@ -1,6 +1,7 @@ #ifndef _ASM_ARM_XEN_PAGE_H #define _ASM_ARM_XEN_PAGE_H +#include #include #include @@ -86,4 +87,7 @@ static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) { return __set_phys_to_machine(pfn, mfn); } + +#define xen_remap(cookie, size) __arm_ioremap((cookie), (size), MT_MEMORY); + #endif /* _ASM_ARM_XEN_PAGE_H */ diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h index 472b9b783019..6aef9fbc09b7 100644 --- a/arch/x86/include/asm/xen/page.h +++ b/arch/x86/include/asm/xen/page.h @@ -212,4 +212,6 @@ unsigned long arbitrary_virt_to_mfn(void *vaddr); void make_lowmem_page_readonly(void *vaddr); void make_lowmem_page_readwrite(void *vaddr); +#define xen_remap(cookie, size) ioremap((cookie), (size)); + #endif /* _ASM_X86_XEN_PAGE_H */ diff --git a/drivers/tty/hvc/hvc_xen.c b/drivers/tty/hvc/hvc_xen.c index 19843ec3f80a..682210d778bd 100644 --- a/drivers/tty/hvc/hvc_xen.c +++ b/drivers/tty/hvc/hvc_xen.c @@ -230,7 +230,7 @@ static int xen_hvm_console_init(void) if (r < 0 || v == 0) goto err; mfn = v; - info->intf = ioremap(mfn << PAGE_SHIFT, PAGE_SIZE); + info->intf = xen_remap(mfn << PAGE_SHIFT, PAGE_SIZE); if (info->intf == NULL) goto err; info->vtermno = HVC_COOKIE; diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 157c0ccda3ef..04c1b2d9b775 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -1147,7 +1147,7 @@ static int gnttab_setup(void) return gnttab_map(0, nr_grant_frames - 1); if (gnttab_shared.addr == NULL) { - gnttab_shared.addr = ioremap(xen_hvm_resume_frames, + gnttab_shared.addr = xen_remap(xen_hvm_resume_frames, PAGE_SIZE * max_nr_gframes); if (gnttab_shared.addr == NULL) { printk(KERN_WARNING diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index 038b71dbf03c..3325884c693f 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -769,7 +769,7 @@ static int __init xenbus_init(void) goto out_error; xen_store_mfn = (unsigned long)v; xen_store_interface = - ioremap(xen_store_mfn << PAGE_SHIFT, PAGE_SIZE); + xen_remap(xen_store_mfn << PAGE_SHIFT, PAGE_SIZE); break; default: pr_warn("Xenstore state unknown\n"); -- cgit v1.2.3 From 76eaca031f0af2bb303e405986f637811956a422 Mon Sep 17 00:00:00 2001 From: Stefan Bader Date: Fri, 15 Feb 2013 09:48:52 +0100 Subject: xen: Send spinlock IPI to all waiters There is a loophole between Xen's current implementation of pv-spinlocks and the scheduler. This was triggerable through a testcase until v3.6 changed the TLB flushing code. The problem potentially is still there just not observable in the same way. What could happen was (is): 1. CPU n tries to schedule task x away and goes into a slow wait for the runq lock of CPU n-# (must be one with a lower number). 2. CPU n-#, while processing softirqs, tries to balance domains and goes into a slow wait for its own runq lock (for updating some records). Since this is a spin_lock_irqsave in softirq context, interrupts will be re-enabled for the duration of the poll_irq hypercall used by Xen. 3. Before the runq lock of CPU n-# is unlocked, CPU n-1 receives an interrupt (e.g. endio) and when processing the interrupt, tries to wake up task x. But that is in schedule and still on_cpu, so try_to_wake_up goes into a tight loop. 4. The runq lock of CPU n-# gets unlocked, but the message only gets sent to the first waiter, which is CPU n-# and that is busily stuck. 5. CPU n-# never returns from the nested interruption to take and release the lock because the scheduler uses a busy wait. And CPU n never finishes the task migration because the unlock notification only went to CPU n-#. To avoid this and since the unlocking code has no real sense of which waiter is best suited to grab the lock, just send the IPI to all of them. This causes the waiters to return from the hyper- call (those not interrupted at least) and do active spinlocking. BugLink: http://bugs.launchpad.net/bugs/1011792 Acked-by: Jan Beulich Signed-off-by: Stefan Bader Cc: stable@vger.kernel.org Signed-off-by: Konrad Rzeszutek Wilk --- arch/x86/xen/spinlock.c | 1 - 1 file changed, 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 83e866d714ce..f7a080ef0354 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -328,7 +328,6 @@ static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) if (per_cpu(lock_spinners, cpu) == xl) { ADD_STATS(released_slow_kicked, 1); xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); - break; } } } -- cgit v1.2.3 From c81611c4e96f595a80d8be9367c385d2c116428b Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 20 Feb 2013 11:48:06 +0000 Subject: xen: event channel arrays are xen_ulong_t and not unsigned long On ARM we want these to be the same size on 32- and 64-bit. This is an ABI change on ARM. X86 does not change. Signed-off-by: Ian Campbell Cc: Jan Beulich Cc: Keir (Xen.org) Cc: Tim Deegan Cc: Stefano Stabellini Cc: linux-arm-kernel@lists.infradead.org Cc: xen-devel@lists.xen.org Cc: Konrad Rzeszutek Wilk Signed-off-by: Konrad Rzeszutek Wilk --- arch/arm/include/asm/xen/events.h | 22 ++++++++ arch/x86/include/asm/xen/events.h | 3 + drivers/xen/events.c | 115 ++++++++++++++++++++++---------------- include/xen/interface/xen.h | 8 +-- 4 files changed, 96 insertions(+), 52 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/xen/events.h b/arch/arm/include/asm/xen/events.h index 94b4e9020b02..5c27696de14f 100644 --- a/arch/arm/include/asm/xen/events.h +++ b/arch/arm/include/asm/xen/events.h @@ -15,4 +15,26 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) return raw_irqs_disabled_flags(regs->ARM_cpsr); } +/* + * We cannot use xchg because it does not support 8-byte + * values. However it is safe to use {ldr,dtd}exd directly because all + * platforms which Xen can run on support those instructions. + */ +static inline xen_ulong_t xchg_xen_ulong(xen_ulong_t *ptr, xen_ulong_t val) +{ + xen_ulong_t oldval; + unsigned int tmp; + + wmb(); + asm volatile("@ xchg_xen_ulong\n" + "1: ldrexd %0, %H0, [%3]\n" + " strexd %1, %2, %H2, [%3]\n" + " teq %1, #0\n" + " bne 1b" + : "=&r" (oldval), "=&r" (tmp) + : "r" (val), "r" (ptr) + : "memory", "cc"); + return oldval; +} + #endif /* _ASM_ARM_XEN_EVENTS_H */ diff --git a/arch/x86/include/asm/xen/events.h b/arch/x86/include/asm/xen/events.h index cc146d51449e..ca842f2769ef 100644 --- a/arch/x86/include/asm/xen/events.h +++ b/arch/x86/include/asm/xen/events.h @@ -16,4 +16,7 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) return raw_irqs_disabled_flags(regs->flags); } +/* No need for a barrier -- XCHG is a barrier on x86. */ +#define xchg_xen_ulong(ptr, val) xchg((ptr), (val)) + #endif /* _ASM_X86_XEN_EVENTS_H */ diff --git a/drivers/xen/events.c b/drivers/xen/events.c index 0be4df39e953..342edc0495cc 100644 --- a/drivers/xen/events.c +++ b/drivers/xen/events.c @@ -120,7 +120,22 @@ static unsigned long *pirq_eoi_map; #endif static bool (*pirq_needs_eoi)(unsigned irq); -static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS/BITS_PER_LONG], +/* + * Note sizeof(xen_ulong_t) can be more than sizeof(unsigned long). Be + * careful to only use bitops which allow for this (e.g + * test_bit/find_first_bit and friends but not __ffs) and to pass + * BITS_PER_EVTCHN_WORD as the bitmask length. + */ +#define BITS_PER_EVTCHN_WORD (sizeof(xen_ulong_t)*8) +/* + * Make a bitmask (i.e. unsigned long *) of a xen_ulong_t + * array. Primarily to avoid long lines (hence the terse name). + */ +#define BM(x) (unsigned long *)(x) +/* Find the first set bit in a evtchn mask */ +#define EVTCHN_FIRST_BIT(w) find_first_bit(BM(&(w)), BITS_PER_EVTCHN_WORD) + +static DEFINE_PER_CPU(xen_ulong_t [NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD], cpu_evtchn_mask); /* Xen will never allocate port zero for any purpose. */ @@ -294,9 +309,9 @@ static bool pirq_needs_eoi_flag(unsigned irq) return info->u.pirq.flags & PIRQ_NEEDS_EOI; } -static inline unsigned long active_evtchns(unsigned int cpu, - struct shared_info *sh, - unsigned int idx) +static inline xen_ulong_t active_evtchns(unsigned int cpu, + struct shared_info *sh, + unsigned int idx) { return sh->evtchn_pending[idx] & per_cpu(cpu_evtchn_mask, cpu)[idx] & @@ -312,8 +327,8 @@ static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu) cpumask_copy(irq_to_desc(irq)->irq_data.affinity, cpumask_of(cpu)); #endif - clear_bit(chn, per_cpu(cpu_evtchn_mask, cpu_from_irq(irq))); - set_bit(chn, per_cpu(cpu_evtchn_mask, cpu)); + clear_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu_from_irq(irq)))); + set_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu))); info_for_irq(irq)->cpu = cpu; } @@ -339,19 +354,19 @@ static void init_evtchn_cpu_bindings(void) static inline void clear_evtchn(int port) { struct shared_info *s = HYPERVISOR_shared_info; - sync_clear_bit(port, &s->evtchn_pending[0]); + sync_clear_bit(port, BM(&s->evtchn_pending[0])); } static inline void set_evtchn(int port) { struct shared_info *s = HYPERVISOR_shared_info; - sync_set_bit(port, &s->evtchn_pending[0]); + sync_set_bit(port, BM(&s->evtchn_pending[0])); } static inline int test_evtchn(int port) { struct shared_info *s = HYPERVISOR_shared_info; - return sync_test_bit(port, &s->evtchn_pending[0]); + return sync_test_bit(port, BM(&s->evtchn_pending[0])); } @@ -375,7 +390,7 @@ EXPORT_SYMBOL_GPL(notify_remote_via_irq); static void mask_evtchn(int port) { struct shared_info *s = HYPERVISOR_shared_info; - sync_set_bit(port, &s->evtchn_mask[0]); + sync_set_bit(port, BM(&s->evtchn_mask[0])); } static void unmask_evtchn(int port) @@ -389,7 +404,7 @@ static void unmask_evtchn(int port) if (unlikely((cpu != cpu_from_evtchn(port)))) do_hypercall = 1; else - evtchn_pending = sync_test_bit(port, &s->evtchn_pending[0]); + evtchn_pending = sync_test_bit(port, BM(&s->evtchn_pending[0])); if (unlikely(evtchn_pending && xen_hvm_domain())) do_hypercall = 1; @@ -403,7 +418,7 @@ static void unmask_evtchn(int port) } else { struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); - sync_clear_bit(port, &s->evtchn_mask[0]); + sync_clear_bit(port, BM(&s->evtchn_mask[0])); /* * The following is basically the equivalent of @@ -411,8 +426,8 @@ static void unmask_evtchn(int port) * the interrupt edge' if the channel is masked. */ if (evtchn_pending && - !sync_test_and_set_bit(port / BITS_PER_LONG, - &vcpu_info->evtchn_pending_sel)) + !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD, + BM(&vcpu_info->evtchn_pending_sel))) vcpu_info->evtchn_upcall_pending = 1; } @@ -1189,7 +1204,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id) { struct shared_info *sh = HYPERVISOR_shared_info; int cpu = smp_processor_id(); - unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu); + xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu); int i; unsigned long flags; static DEFINE_SPINLOCK(debug_lock); @@ -1205,7 +1220,7 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id) pending = (get_irq_regs() && i == cpu) ? xen_irqs_disabled(get_irq_regs()) : v->evtchn_upcall_mask; - printk("%d: masked=%d pending=%d event_sel %0*lx\n ", i, + printk("%d: masked=%d pending=%d event_sel %0*"PRI_xen_ulong"\n ", i, pending, v->evtchn_upcall_pending, (int)(sizeof(v->evtchn_pending_sel)*2), v->evtchn_pending_sel); @@ -1214,49 +1229,52 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id) printk("\npending:\n "); for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--) - printk("%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2, + printk("%0*"PRI_xen_ulong"%s", + (int)sizeof(sh->evtchn_pending[0])*2, sh->evtchn_pending[i], i % 8 == 0 ? "\n " : " "); printk("\nglobal mask:\n "); for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) - printk("%0*lx%s", + printk("%0*"PRI_xen_ulong"%s", (int)(sizeof(sh->evtchn_mask[0])*2), sh->evtchn_mask[i], i % 8 == 0 ? "\n " : " "); printk("\nglobally unmasked:\n "); for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) - printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2), + printk("%0*"PRI_xen_ulong"%s", + (int)(sizeof(sh->evtchn_mask[0])*2), sh->evtchn_pending[i] & ~sh->evtchn_mask[i], i % 8 == 0 ? "\n " : " "); printk("\nlocal cpu%d mask:\n ", cpu); - for (i = (NR_EVENT_CHANNELS/BITS_PER_LONG)-1; i >= 0; i--) - printk("%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2), + for (i = (NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--) + printk("%0*"PRI_xen_ulong"%s", (int)(sizeof(cpu_evtchn[0])*2), cpu_evtchn[i], i % 8 == 0 ? "\n " : " "); printk("\nlocally unmasked:\n "); for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) { - unsigned long pending = sh->evtchn_pending[i] + xen_ulong_t pending = sh->evtchn_pending[i] & ~sh->evtchn_mask[i] & cpu_evtchn[i]; - printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2), + printk("%0*"PRI_xen_ulong"%s", + (int)(sizeof(sh->evtchn_mask[0])*2), pending, i % 8 == 0 ? "\n " : " "); } printk("\npending list:\n"); for (i = 0; i < NR_EVENT_CHANNELS; i++) { - if (sync_test_bit(i, sh->evtchn_pending)) { - int word_idx = i / BITS_PER_LONG; + if (sync_test_bit(i, BM(sh->evtchn_pending))) { + int word_idx = i / BITS_PER_EVTCHN_WORD; printk(" %d: event %d -> irq %d%s%s%s\n", cpu_from_evtchn(i), i, evtchn_to_irq[i], - sync_test_bit(word_idx, &v->evtchn_pending_sel) + sync_test_bit(word_idx, BM(&v->evtchn_pending_sel)) ? "" : " l2-clear", - !sync_test_bit(i, sh->evtchn_mask) + !sync_test_bit(i, BM(sh->evtchn_mask)) ? "" : " globally-masked", - sync_test_bit(i, cpu_evtchn) + sync_test_bit(i, BM(cpu_evtchn)) ? "" : " locally-masked"); } } @@ -1273,7 +1291,7 @@ static DEFINE_PER_CPU(unsigned int, current_bit_idx); /* * Mask out the i least significant bits of w */ -#define MASK_LSBS(w, i) (w & ((~0UL) << i)) +#define MASK_LSBS(w, i) (w & ((~((xen_ulong_t)0UL)) << i)) /* * Search the CPUs pending events bitmasks. For each one found, map @@ -1295,18 +1313,19 @@ static void __xen_evtchn_do_upcall(void) unsigned count; do { - unsigned long pending_words; + xen_ulong_t pending_words; vcpu_info->evtchn_upcall_pending = 0; if (__this_cpu_inc_return(xed_nesting_count) - 1) goto out; -#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ - /* Clear master flag /before/ clearing selector flag. */ - wmb(); -#endif - pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0); + /* + * Master flag must be cleared /before/ clearing + * selector flag. xchg_xen_ulong must contain an + * appropriate barrier. + */ + pending_words = xchg_xen_ulong(&vcpu_info->evtchn_pending_sel, 0); start_word_idx = __this_cpu_read(current_word_idx); start_bit_idx = __this_cpu_read(current_bit_idx); @@ -1314,8 +1333,8 @@ static void __xen_evtchn_do_upcall(void) word_idx = start_word_idx; for (i = 0; pending_words != 0; i++) { - unsigned long pending_bits; - unsigned long words; + xen_ulong_t pending_bits; + xen_ulong_t words; words = MASK_LSBS(pending_words, word_idx); @@ -1327,7 +1346,7 @@ static void __xen_evtchn_do_upcall(void) bit_idx = 0; continue; } - word_idx = __ffs(words); + word_idx = EVTCHN_FIRST_BIT(words); pending_bits = active_evtchns(cpu, s, word_idx); bit_idx = 0; /* usually scan entire word from start */ @@ -1342,7 +1361,7 @@ static void __xen_evtchn_do_upcall(void) } do { - unsigned long bits; + xen_ulong_t bits; int port, irq; struct irq_desc *desc; @@ -1352,10 +1371,10 @@ static void __xen_evtchn_do_upcall(void) if (bits == 0) break; - bit_idx = __ffs(bits); + bit_idx = EVTCHN_FIRST_BIT(bits); /* Process port. */ - port = (word_idx * BITS_PER_LONG) + bit_idx; + port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx; irq = evtchn_to_irq[port]; if (irq != -1) { @@ -1364,12 +1383,12 @@ static void __xen_evtchn_do_upcall(void) generic_handle_irq_desc(irq, desc); } - bit_idx = (bit_idx + 1) % BITS_PER_LONG; + bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD; /* Next caller starts at last processed + 1 */ __this_cpu_write(current_word_idx, bit_idx ? word_idx : - (word_idx+1) % BITS_PER_LONG); + (word_idx+1) % BITS_PER_EVTCHN_WORD); __this_cpu_write(current_bit_idx, bit_idx); } while (bit_idx != 0); @@ -1377,7 +1396,7 @@ static void __xen_evtchn_do_upcall(void) if ((word_idx != start_word_idx) || (i != 0)) pending_words &= ~(1UL << word_idx); - word_idx = (word_idx + 1) % BITS_PER_LONG; + word_idx = (word_idx + 1) % BITS_PER_EVTCHN_WORD; } BUG_ON(!irqs_disabled()); @@ -1487,8 +1506,8 @@ int resend_irq_on_evtchn(unsigned int irq) if (!VALID_EVTCHN(evtchn)) return 1; - masked = sync_test_and_set_bit(evtchn, s->evtchn_mask); - sync_set_bit(evtchn, s->evtchn_pending); + masked = sync_test_and_set_bit(evtchn, BM(s->evtchn_mask)); + sync_set_bit(evtchn, BM(s->evtchn_pending)); if (!masked) unmask_evtchn(evtchn); @@ -1536,8 +1555,8 @@ static int retrigger_dynirq(struct irq_data *data) if (VALID_EVTCHN(evtchn)) { int masked; - masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask); - sync_set_bit(evtchn, sh->evtchn_pending); + masked = sync_test_and_set_bit(evtchn, BM(sh->evtchn_mask)); + sync_set_bit(evtchn, BM(sh->evtchn_pending)); if (!masked) unmask_evtchn(evtchn); ret = 1; diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h index 886a5d80a18f..53ec4167bd0b 100644 --- a/include/xen/interface/xen.h +++ b/include/xen/interface/xen.h @@ -285,7 +285,7 @@ DEFINE_GUEST_HANDLE_STRUCT(multicall_entry); * Event channel endpoints per domain: * 1024 if a long is 32 bits; 4096 if a long is 64 bits. */ -#define NR_EVENT_CHANNELS (sizeof(unsigned long) * sizeof(unsigned long) * 64) +#define NR_EVENT_CHANNELS (sizeof(xen_ulong_t) * sizeof(xen_ulong_t) * 64) struct vcpu_time_info { /* @@ -341,7 +341,7 @@ struct vcpu_info { */ uint8_t evtchn_upcall_pending; uint8_t evtchn_upcall_mask; - unsigned long evtchn_pending_sel; + xen_ulong_t evtchn_pending_sel; struct arch_vcpu_info arch; struct pvclock_vcpu_time_info time; }; /* 64 bytes (x86) */ @@ -384,8 +384,8 @@ struct shared_info { * per-vcpu selector word to be set. Each bit in the selector covers a * 'C long' in the PENDING bitfield array. */ - unsigned long evtchn_pending[sizeof(unsigned long) * 8]; - unsigned long evtchn_mask[sizeof(unsigned long) * 8]; + xen_ulong_t evtchn_pending[sizeof(xen_ulong_t) * 8]; + xen_ulong_t evtchn_mask[sizeof(xen_ulong_t) * 8]; /* * Wallclock time: updated only by control software. Guests should base -- cgit v1.2.3