summaryrefslogtreecommitdiffstats
path: root/arch/s390/lib
diff options
context:
space:
mode:
authorMartin Schwidefsky <schwidefsky@de.ibm.com>2017-03-24 17:32:23 +0100
committerMartin Schwidefsky <schwidefsky@de.ibm.com>2017-09-28 07:29:44 +0200
commiteb3b7b848fb3dd00f7a57d633d4ae4d194aa7865 (patch)
treeab510aeb7e5aedc07d1cd0e60c5d430c74e02bd8 /arch/s390/lib
parentb96f7d881ad94203e997cd2aa7112d4a06d121ef (diff)
downloadlinux-eb3b7b848fb3dd00f7a57d633d4ae4d194aa7865.tar.bz2
s390/rwlock: introduce rwlock wait queueing
Like the common queued rwlock code the s390 implementation uses the queued spinlock code on a spinlock_t embedded in the rwlock_t to achieve the queueing. The encoding of the rwlock_t differs though, the counter field in the rwlock_t is split into two parts. The upper two bytes hold the write bit and the write wait counter, the lower two bytes hold the read counter. The arch_read_lock operation works exactly like the common qrwlock but the enqueue operation for a writer follows a diffent logic. After the failed inline try to get the rwlock in write, the writer first increases the write wait counter, acquires the wait spin_lock for the queueing, and then loops until there are no readers and the write bit is zero. Without the write wait counter a CPU that just released the rwlock could immediately reacquire the lock in the inline code, bypassing all outstanding read and write waiters. For s390 this would cause massive imbalances in favour of writers in case of a contended rwlock. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Diffstat (limited to 'arch/s390/lib')
-rw-r--r--arch/s390/lib/spinlock.c138
1 files changed, 29 insertions, 109 deletions
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 6747134227cd..43b0d46c3786 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -268,129 +268,49 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp)
}
EXPORT_SYMBOL(arch_spin_trylock_retry);
-void _raw_read_lock_wait(arch_rwlock_t *rw)
+void arch_read_lock_wait(arch_rwlock_t *rw)
{
- int count = spin_retry;
- int owner, old;
-
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
- __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
-#endif
- owner = 0;
- while (1) {
- if (count-- <= 0) {
- if (owner && arch_vcpu_is_preempted(owner - 1))
- smp_yield_cpu(owner - 1);
- count = spin_retry;
- }
- old = ACCESS_ONCE(rw->lock);
- owner = ACCESS_ONCE(rw->owner);
- if (old < 0)
- continue;
- if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
- return;
+ if (unlikely(in_interrupt())) {
+ while (READ_ONCE(rw->cnts) & 0x10000)
+ barrier();
+ return;
}
-}
-EXPORT_SYMBOL(_raw_read_lock_wait);
-
-int _raw_read_trylock_retry(arch_rwlock_t *rw)
-{
- int count = spin_retry;
- int old;
- while (count-- > 0) {
- old = ACCESS_ONCE(rw->lock);
- if (old < 0)
- continue;
- if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
- return 1;
- }
- return 0;
+ /* Remove this reader again to allow recursive read locking */
+ __atomic_add_const(-1, &rw->cnts);
+ /* Put the reader into the wait queue */
+ arch_spin_lock(&rw->wait);
+ /* Now add this reader to the count value again */
+ __atomic_add_const(1, &rw->cnts);
+ /* Loop until the writer is done */
+ while (READ_ONCE(rw->cnts) & 0x10000)
+ barrier();
+ arch_spin_unlock(&rw->wait);
}
-EXPORT_SYMBOL(_raw_read_trylock_retry);
+EXPORT_SYMBOL(arch_read_lock_wait);
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-
-void _raw_write_lock_wait(arch_rwlock_t *rw, int prev)
+void arch_write_lock_wait(arch_rwlock_t *rw)
{
- int count = spin_retry;
- int owner, old;
-
- owner = 0;
- while (1) {
- if (count-- <= 0) {
- if (owner && arch_vcpu_is_preempted(owner - 1))
- smp_yield_cpu(owner - 1);
- count = spin_retry;
- }
- old = ACCESS_ONCE(rw->lock);
- owner = ACCESS_ONCE(rw->owner);
- smp_mb();
- if (old >= 0) {
- prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
- old = prev;
- }
- if ((old & 0x7fffffff) == 0 && prev >= 0)
- break;
- }
-}
-EXPORT_SYMBOL(_raw_write_lock_wait);
+ int old;
-#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+ /* Add this CPU to the write waiters */
+ __atomic_add(0x20000, &rw->cnts);
-void _raw_write_lock_wait(arch_rwlock_t *rw)
-{
- int count = spin_retry;
- int owner, old, prev;
+ /* Put the writer into the wait queue */
+ arch_spin_lock(&rw->wait);
- prev = 0x80000000;
- owner = 0;
while (1) {
- if (count-- <= 0) {
- if (owner && arch_vcpu_is_preempted(owner - 1))
- smp_yield_cpu(owner - 1);
- count = spin_retry;
- }
- old = ACCESS_ONCE(rw->lock);
- owner = ACCESS_ONCE(rw->owner);
- if (old >= 0 &&
- __atomic_cmpxchg_bool(&rw->lock, old, old | 0x80000000))
- prev = old;
- else
- smp_mb();
- if ((old & 0x7fffffff) == 0 && prev >= 0)
+ old = READ_ONCE(rw->cnts);
+ if ((old & 0x1ffff) == 0 &&
+ __atomic_cmpxchg_bool(&rw->cnts, old, old | 0x10000))
+ /* Got the lock */
break;
+ barrier();
}
-}
-EXPORT_SYMBOL(_raw_write_lock_wait);
-
-#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
-int _raw_write_trylock_retry(arch_rwlock_t *rw)
-{
- int count = spin_retry;
- int old;
-
- while (count-- > 0) {
- old = ACCESS_ONCE(rw->lock);
- if (old)
- continue;
- if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000))
- return 1;
- }
- return 0;
-}
-EXPORT_SYMBOL(_raw_write_trylock_retry);
-void arch_lock_relax(int cpu)
-{
- if (!cpu)
- return;
- if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted(cpu - 1))
- return;
- smp_yield_cpu(cpu - 1);
+ arch_spin_unlock(&rw->wait);
}
-EXPORT_SYMBOL(arch_lock_relax);
+EXPORT_SYMBOL(arch_write_lock_wait);
void arch_spin_relax(arch_spinlock_t *lp)
{