summaryrefslogtreecommitdiffstats
path: root/arch/tile/lib/atomic_32.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/tile/lib/atomic_32.c')
-rw-r--r--arch/tile/lib/atomic_32.c347
1 files changed, 347 insertions, 0 deletions
diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c
new file mode 100644
index 000000000000..be1e8acd105d
--- /dev/null
+++ b/arch/tile/lib/atomic_32.c
@@ -0,0 +1,347 @@
+/*
+ * Copyright 2010 Tilera Corporation. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT. See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/cache.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <asm/atomic.h>
+#include <arch/chip.h>
+
+/* The routines in atomic_asm.S are private, so we only declare them here. */
+extern struct __get_user __atomic_cmpxchg(volatile int *p,
+ int *lock, int o, int n);
+extern struct __get_user __atomic_xchg(volatile int *p, int *lock, int n);
+extern struct __get_user __atomic_xchg_add(volatile int *p, int *lock, int n);
+extern struct __get_user __atomic_xchg_add_unless(volatile int *p,
+ int *lock, int o, int n);
+extern struct __get_user __atomic_or(volatile int *p, int *lock, int n);
+extern struct __get_user __atomic_andn(volatile int *p, int *lock, int n);
+extern struct __get_user __atomic_xor(volatile int *p, int *lock, int n);
+
+extern u64 __atomic64_cmpxchg(volatile u64 *p, int *lock, u64 o, u64 n);
+extern u64 __atomic64_xchg(volatile u64 *p, int *lock, u64 n);
+extern u64 __atomic64_xchg_add(volatile u64 *p, int *lock, u64 n);
+extern u64 __atomic64_xchg_add_unless(volatile u64 *p,
+ int *lock, u64 o, u64 n);
+
+
+/* See <asm/atomic.h> */
+#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
+
+/*
+ * A block of memory containing locks for atomic ops. Each instance of this
+ * struct will be homed on a different CPU.
+ */
+struct atomic_locks_on_cpu {
+ int lock[ATOMIC_HASH_L2_SIZE];
+} __attribute__((aligned(ATOMIC_HASH_L2_SIZE * 4)));
+
+static DEFINE_PER_CPU(struct atomic_locks_on_cpu, atomic_lock_pool);
+
+/* The locks we'll use until __init_atomic_per_cpu is called. */
+static struct atomic_locks_on_cpu __initdata initial_atomic_locks;
+
+/* Hash into this vector to get a pointer to lock for the given atomic. */
+struct atomic_locks_on_cpu *atomic_lock_ptr[ATOMIC_HASH_L1_SIZE]
+ __write_once = {
+ [0 ... ATOMIC_HASH_L1_SIZE-1] (&initial_atomic_locks)
+};
+
+#else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
+
+/* This page is remapped on startup to be hash-for-home. */
+int atomic_locks[PAGE_SIZE / sizeof(int) /* Only ATOMIC_HASH_SIZE is used */]
+ __attribute__((aligned(PAGE_SIZE), section(".bss.page_aligned")));
+
+#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
+
+static inline int *__atomic_hashed_lock(volatile void *v)
+{
+ /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec.S */
+#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
+ unsigned long i =
+ (unsigned long) v & ((PAGE_SIZE-1) & -sizeof(long long));
+ unsigned long n = __insn_crc32_32(0, i);
+
+ /* Grab high bits for L1 index. */
+ unsigned long l1_index = n >> ((sizeof(n) * 8) - ATOMIC_HASH_L1_SHIFT);
+ /* Grab low bits for L2 index. */
+ unsigned long l2_index = n & (ATOMIC_HASH_L2_SIZE - 1);
+
+ return &atomic_lock_ptr[l1_index]->lock[l2_index];
+#else
+ /*
+ * Use bits [3, 3 + ATOMIC_HASH_SHIFT) as the lock index.
+ * Using mm works here because atomic_locks is page aligned.
+ */
+ unsigned long ptr = __insn_mm((unsigned long)v >> 1,
+ (unsigned long)atomic_locks,
+ 2, (ATOMIC_HASH_SHIFT + 2) - 1);
+ return (int *)ptr;
+#endif
+}
+
+#ifdef CONFIG_SMP
+/* Return whether the passed pointer is a valid atomic lock pointer. */
+static int is_atomic_lock(int *p)
+{
+#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
+ int i;
+ for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
+
+ if (p >= &atomic_lock_ptr[i]->lock[0] &&
+ p < &atomic_lock_ptr[i]->lock[ATOMIC_HASH_L2_SIZE]) {
+ return 1;
+ }
+ }
+ return 0;
+#else
+ return p >= &atomic_locks[0] && p < &atomic_locks[ATOMIC_HASH_SIZE];
+#endif
+}
+
+void __atomic_fault_unlock(int *irqlock_word)
+{
+ BUG_ON(!is_atomic_lock(irqlock_word));
+ BUG_ON(*irqlock_word != 1);
+ *irqlock_word = 0;
+}
+
+#endif /* CONFIG_SMP */
+
+static inline int *__atomic_setup(volatile void *v)
+{
+ /* Issue a load to the target to bring it into cache. */
+ *(volatile int *)v;
+ return __atomic_hashed_lock(v);
+}
+
+int _atomic_xchg(atomic_t *v, int n)
+{
+ return __atomic_xchg(&v->counter, __atomic_setup(v), n).val;
+}
+EXPORT_SYMBOL(_atomic_xchg);
+
+int _atomic_xchg_add(atomic_t *v, int i)
+{
+ return __atomic_xchg_add(&v->counter, __atomic_setup(v), i).val;
+}
+EXPORT_SYMBOL(_atomic_xchg_add);
+
+int _atomic_xchg_add_unless(atomic_t *v, int a, int u)
+{
+ /*
+ * Note: argument order is switched here since it is easier
+ * to use the first argument consistently as the "old value"
+ * in the assembly, as is done for _atomic_cmpxchg().
+ */
+ return __atomic_xchg_add_unless(&v->counter, __atomic_setup(v), u, a)
+ .val;
+}
+EXPORT_SYMBOL(_atomic_xchg_add_unless);
+
+int _atomic_cmpxchg(atomic_t *v, int o, int n)
+{
+ return __atomic_cmpxchg(&v->counter, __atomic_setup(v), o, n).val;
+}
+EXPORT_SYMBOL(_atomic_cmpxchg);
+
+unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask)
+{
+ return __atomic_or((int *)p, __atomic_setup(p), mask).val;
+}
+EXPORT_SYMBOL(_atomic_or);
+
+unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask)
+{
+ return __atomic_andn((int *)p, __atomic_setup(p), mask).val;
+}
+EXPORT_SYMBOL(_atomic_andn);
+
+unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask)
+{
+ return __atomic_xor((int *)p, __atomic_setup(p), mask).val;
+}
+EXPORT_SYMBOL(_atomic_xor);
+
+
+u64 _atomic64_xchg(atomic64_t *v, u64 n)
+{
+ return __atomic64_xchg(&v->counter, __atomic_setup(v), n);
+}
+EXPORT_SYMBOL(_atomic64_xchg);
+
+u64 _atomic64_xchg_add(atomic64_t *v, u64 i)
+{
+ return __atomic64_xchg_add(&v->counter, __atomic_setup(v), i);
+}
+EXPORT_SYMBOL(_atomic64_xchg_add);
+
+u64 _atomic64_xchg_add_unless(atomic64_t *v, u64 a, u64 u)
+{
+ /*
+ * Note: argument order is switched here since it is easier
+ * to use the first argument consistently as the "old value"
+ * in the assembly, as is done for _atomic_cmpxchg().
+ */
+ return __atomic64_xchg_add_unless(&v->counter, __atomic_setup(v),
+ u, a);
+}
+EXPORT_SYMBOL(_atomic64_xchg_add_unless);
+
+u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
+{
+ return __atomic64_cmpxchg(&v->counter, __atomic_setup(v), o, n);
+}
+EXPORT_SYMBOL(_atomic64_cmpxchg);
+
+
+static inline int *__futex_setup(__user int *v)
+{
+ /*
+ * Issue a prefetch to the counter to bring it into cache.
+ * As for __atomic_setup, but we can't do a read into the L1
+ * since it might fault; instead we do a prefetch into the L2.
+ */
+ __insn_prefetch(v);
+ return __atomic_hashed_lock(v);
+}
+
+struct __get_user futex_set(int *v, int i)
+{
+ return __atomic_xchg(v, __futex_setup(v), i);
+}
+
+struct __get_user futex_add(int *v, int n)
+{
+ return __atomic_xchg_add(v, __futex_setup(v), n);
+}
+
+struct __get_user futex_or(int *v, int n)
+{
+ return __atomic_or(v, __futex_setup(v), n);
+}
+
+struct __get_user futex_andn(int *v, int n)
+{
+ return __atomic_andn(v, __futex_setup(v), n);
+}
+
+struct __get_user futex_xor(int *v, int n)
+{
+ return __atomic_xor(v, __futex_setup(v), n);
+}
+
+struct __get_user futex_cmpxchg(int *v, int o, int n)
+{
+ return __atomic_cmpxchg(v, __futex_setup(v), o, n);
+}
+
+/*
+ * If any of the atomic or futex routines hit a bad address (not in
+ * the page tables at kernel PL) this routine is called. The futex
+ * routines are never used on kernel space, and the normal atomics and
+ * bitops are never used on user space. So a fault on kernel space
+ * must be fatal, but a fault on userspace is a futex fault and we
+ * need to return -EFAULT. Note that the context this routine is
+ * invoked in is the context of the "_atomic_xxx()" routines called
+ * by the functions in this file.
+ */
+struct __get_user __atomic_bad_address(int *addr)
+{
+ if (unlikely(!access_ok(VERIFY_WRITE, addr, sizeof(int))))
+ panic("Bad address used for kernel atomic op: %p\n", addr);
+ return (struct __get_user) { .err = -EFAULT };
+}
+
+
+#if CHIP_HAS_CBOX_HOME_MAP()
+static int __init noatomichash(char *str)
+{
+ printk("noatomichash is deprecated.\n");
+ return 1;
+}
+__setup("noatomichash", noatomichash);
+#endif
+
+void __init __init_atomic_per_cpu(void)
+{
+#if ATOMIC_LOCKS_FOUND_VIA_TABLE()
+
+ unsigned int i;
+ int actual_cpu;
+
+ /*
+ * Before this is called from setup, we just have one lock for
+ * all atomic objects/operations. Here we replace the
+ * elements of atomic_lock_ptr so that they point at per_cpu
+ * integers. This seemingly over-complex approach stems from
+ * the fact that DEFINE_PER_CPU defines an entry for each cpu
+ * in the grid, not each cpu from 0..ATOMIC_HASH_SIZE-1. But
+ * for efficient hashing of atomics to their locks we want a
+ * compile time constant power of 2 for the size of this
+ * table, so we use ATOMIC_HASH_SIZE.
+ *
+ * Here we populate atomic_lock_ptr from the per cpu
+ * atomic_lock_pool, interspersing by actual cpu so that
+ * subsequent elements are homed on consecutive cpus.
+ */
+
+ actual_cpu = cpumask_first(cpu_possible_mask);
+
+ for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
+ /*
+ * Preincrement to slightly bias against using cpu 0,
+ * which has plenty of stuff homed on it already.
+ */
+ actual_cpu = cpumask_next(actual_cpu, cpu_possible_mask);
+ if (actual_cpu >= nr_cpu_ids)
+ actual_cpu = cpumask_first(cpu_possible_mask);
+
+ atomic_lock_ptr[i] = &per_cpu(atomic_lock_pool, actual_cpu);
+ }
+
+#else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
+
+ /* Validate power-of-two and "bigger than cpus" assumption */
+ BUG_ON(ATOMIC_HASH_SIZE & (ATOMIC_HASH_SIZE-1));
+ BUG_ON(ATOMIC_HASH_SIZE < nr_cpu_ids);
+
+ /*
+ * On TILEPro we prefer to use a single hash-for-home
+ * page, since this means atomic operations are less
+ * likely to encounter a TLB fault and thus should
+ * in general perform faster. You may wish to disable
+ * this in situations where few hash-for-home tiles
+ * are configured.
+ */
+ BUG_ON((unsigned long)atomic_locks % PAGE_SIZE != 0);
+
+ /* The locks must all fit on one page. */
+ BUG_ON(ATOMIC_HASH_SIZE * sizeof(int) > PAGE_SIZE);
+
+ /*
+ * We use the page offset of the atomic value's address as
+ * an index into atomic_locks, excluding the low 3 bits.
+ * That should not produce more indices than ATOMIC_HASH_SIZE.
+ */
+ BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
+
+#endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
+
+ /* The futex code makes this assumption, so we validate it here. */
+ BUG_ON(sizeof(atomic_t) != sizeof(int));
+}