diff options
Diffstat (limited to 'arch')
697 files changed, 13140 insertions, 19613 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 1aa59063f1fd..d1f2ed462ac8 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -13,6 +13,9 @@ config KEXEC_CORE config HAVE_IMA_KEXEC bool +config HOTPLUG_SMT + bool + config OPROFILE tristate "OProfile system profiling" depends on PROFILING diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 767bfdd42992..150a1c5d6a2c 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -18,11 +18,11 @@ * To ensure dependency ordering is preserved for the _relaxed and * _release atomics, an smp_read_barrier_depends() is unconditionally * inserted into the _relaxed variants, which are used to build the - * barriered versions. To avoid redundant back-to-back fences, we can - * define the _acquire and _fence versions explicitly. + * barriered versions. Avoid redundant back-to-back fences in the + * _acquire and _fence versions. */ -#define __atomic_op_acquire(op, args...) op##_relaxed(args) -#define __atomic_op_fence __atomic_op_release +#define __atomic_acquire_fence() +#define __atomic_post_full_fence() #define ATOMIC_INIT(i) { (i) } #define ATOMIC64_INIT(i) { (i) } @@ -206,7 +206,7 @@ ATOMIC_OPS(xor, xor) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) /** - * __atomic_add_unless - add unless the number is a given value + * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -214,7 +214,7 @@ ATOMIC_OPS(xor, xor) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) +static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int c, new, old; smp_mb(); @@ -235,38 +235,39 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) smp_mb(); return old; } - +#define atomic_fetch_add_unless atomic_fetch_add_unless /** - * atomic64_add_unless - add unless the number is a given value + * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. * * Atomically adds @a to @v, so long as it was not @u. - * Returns true iff @v was not @u. + * Returns the old value of @v. */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +static __inline__ long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) { - long c, tmp; + long c, new, old; smp_mb(); __asm__ __volatile__( - "1: ldq_l %[tmp],%[mem]\n" - " cmpeq %[tmp],%[u],%[c]\n" - " addq %[tmp],%[a],%[tmp]\n" + "1: ldq_l %[old],%[mem]\n" + " cmpeq %[old],%[u],%[c]\n" + " addq %[old],%[a],%[new]\n" " bne %[c],2f\n" - " stq_c %[tmp],%[mem]\n" - " beq %[tmp],3f\n" + " stq_c %[new],%[mem]\n" + " beq %[new],3f\n" "2:\n" ".subsection 2\n" "3: br 1b\n" ".previous" - : [tmp] "=&r"(tmp), [c] "=&r"(c) + : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u) : "memory"); smp_mb(); - return !c; + return old; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless /* * atomic64_dec_if_positive - decrement by 1 if old value positive @@ -295,31 +296,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) smp_mb(); return old - 1; } - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) - -#define atomic_dec_return(v) atomic_sub_return(1,(v)) -#define atomic64_dec_return(v) atomic64_sub_return(1,(v)) - -#define atomic_inc_return(v) atomic_add_return(1,(v)) -#define atomic64_inc_return(v) atomic64_add_return(1,(v)) - -#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0) - -#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) -#define atomic64_inc_and_test(v) (atomic64_add_return(1, (v)) == 0) - -#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) -#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) - -#define atomic_inc(v) atomic_add(1,(v)) -#define atomic64_inc(v) atomic64_add(1,(v)) - -#define atomic_dec(v) atomic_sub(1,(v)) -#define atomic64_dec(v) atomic64_sub(1,(v)) +#define atomic64_dec_if_positive atomic64_dec_if_positive #endif /* _ALPHA_ATOMIC_H */ diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 6e921754c8fc..c210a25dd6da 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1180,13 +1180,10 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) SYSCALL_DEFINE4(osf_wait4, pid_t, pid, int __user *, ustatus, int, options, struct rusage32 __user *, ur) { - unsigned int status = 0; struct rusage r; - long err = kernel_wait4(pid, &status, options, &r); + long err = kernel_wait4(pid, ustatus, options, &r); if (err <= 0) return err; - if (put_user(status, ustatus)) - return -EFAULT; if (!ur) return err; if (put_tv_to_tv32(&ur->ru_utime, &r.ru_utime)) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index e81bcd271be7..5151d81476a1 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -50,6 +50,9 @@ config ARC select HAVE_KERNEL_LZMA select ARCH_HAS_PTE_SPECIAL +config ARCH_HAS_CACHE_LINE_SIZE + def_bool y + config MIGHT_HAVE_PCI bool @@ -413,7 +416,7 @@ config ARC_HAS_DIV_REM config ARC_HAS_ACCL_REGS bool "Reg Pair ACCL:ACCH (FPU and/or MPY > 6)" - default n + default y help Depending on the configuration, CPU can contain accumulator reg-pair (also referred to as r58:r59). These can also be used by gcc as GPR so diff --git a/arch/arc/Makefile b/arch/arc/Makefile index d37f49d6a27f..6c1b20dd76ad 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -16,7 +16,7 @@ endif KBUILD_DEFCONFIG := nsim_700_defconfig -cflags-y += -fno-common -pipe -fno-builtin -D__linux__ +cflags-y += -fno-common -pipe -fno-builtin -mmedium-calls -D__linux__ cflags-$(CONFIG_ISA_ARCOMPACT) += -mA7 cflags-$(CONFIG_ISA_ARCV2) += -mcpu=archs @@ -140,16 +140,3 @@ dtbs: scripts archclean: $(Q)$(MAKE) $(clean)=$(boot) - -# Hacks to enable final link due to absence of link-time branch relexation -# and gcc choosing optimal(shorter) branches at -O3 -# -# vineetg Feb 2010: -mlong-calls switched off for overall kernel build -# However lib/decompress_inflate.o (.init.text) calls -# zlib_inflate_workspacesize (.text) causing relocation errors. -# Thus forcing all exten calls in this file to be long calls -export CFLAGS_decompress_inflate.o = -mmedium-calls -export CFLAGS_initramfs.o = -mmedium-calls -ifdef CONFIG_SMP -export CFLAGS_core.o = -mmedium-calls -endif diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig index 09f85154c5a4..a635ea972304 100644 --- a/arch/arc/configs/axs101_defconfig +++ b/arch/arc/configs/axs101_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs/" CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig index 09fed3ef22b6..aa507e423075 100644 --- a/arch/arc/configs/axs103_defconfig +++ b/arch/arc/configs/axs103_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index ea2f6d817d1a..eba07f468654 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig index ab231c040efe..098b19fbaa51 100644 --- a/arch/arc/configs/haps_hs_defconfig +++ b/arch/arc/configs/haps_hs_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_EXPERT=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig index cf449cbf440d..0104c404d897 100644 --- a/arch/arc/configs/haps_hs_smp_defconfig +++ b/arch/arc/configs/haps_hs_smp_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig index 1b54c72f4296..6491be0ddbc9 100644 --- a/arch/arc/configs/hsdk_defconfig +++ b/arch/arc/configs/hsdk_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig index 31c2c70b34a1..99e05cf63fca 100644 --- a/arch/arc/configs/nsim_700_defconfig +++ b/arch/arc/configs/nsim_700_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs/" CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsim_hs_defconfig b/arch/arc/configs/nsim_hs_defconfig index a578c721d50f..0dc4f9b737e7 100644 --- a/arch/arc/configs/nsim_hs_defconfig +++ b/arch/arc/configs/nsim_hs_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../../arc_initramfs_hs/" CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsim_hs_smp_defconfig b/arch/arc/configs/nsim_hs_smp_defconfig index 37d7395f3272..be3c30a15e54 100644 --- a/arch/arc/configs/nsim_hs_smp_defconfig +++ b/arch/arc/configs/nsim_hs_smp_defconfig @@ -9,7 +9,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs_hs/" CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig index 1e1470e2a7f0..3a74b9b21772 100644 --- a/arch/arc/configs/nsimosci_defconfig +++ b/arch/arc/configs/nsimosci_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs/" CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig index 084a6e42685b..ea2834b4dc1d 100644 --- a/arch/arc/configs/nsimosci_hs_defconfig +++ b/arch/arc/configs/nsimosci_hs_defconfig @@ -11,7 +11,6 @@ CONFIG_NAMESPACES=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs_hs/" CONFIG_KALLSYMS_ALL=y CONFIG_EMBEDDED=y CONFIG_PERF_EVENTS=y diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig index f36d47990415..80a5a1b4924b 100644 --- a/arch/arc/configs/nsimosci_hs_smp_defconfig +++ b/arch/arc/configs/nsimosci_hs_smp_defconfig @@ -9,7 +9,6 @@ CONFIG_IKCONFIG_PROC=y # CONFIG_UTS_NS is not set # CONFIG_PID_NS is not set CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_SOURCE="../arc_initramfs_hs/" CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set CONFIG_KPROBES=y diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index 1aca2e8fd1ba..2cc87f909747 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -56,7 +56,6 @@ CONFIG_STMMAC_ETH=y # CONFIG_INPUT is not set # CONFIG_SERIO is not set # CONFIG_VT is not set -CONFIG_DEVPTS_MULTIPLE_INSTANCES=y # CONFIG_LEGACY_PTYS is not set # CONFIG_DEVKMEM is not set CONFIG_SERIAL_8250=y diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 11859287c52a..4e0072730241 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -187,7 +187,8 @@ static inline int atomic_fetch_##op(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define atomic_andnot atomic_andnot +#define atomic_andnot atomic_andnot +#define atomic_fetch_andnot atomic_fetch_andnot #undef ATOMIC_OPS #define ATOMIC_OPS(op, c_op, asm_op) \ @@ -296,8 +297,6 @@ ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) ATOMIC_FETCH_OP(op, c_op, asm_op) ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) -#define atomic_andnot(mask, v) atomic_and(~(mask), (v)) -#define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v)) ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) @@ -308,48 +307,6 @@ ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -/** - * __atomic_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v - */ -#define __atomic_add_unless(v, a, u) \ -({ \ - int c, old; \ - \ - /* \ - * Explicit full memory barrier needed before/after as \ - * LLOCK/SCOND thmeselves don't provide any such semantics \ - */ \ - smp_mb(); \ - \ - c = atomic_read(v); \ - while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\ - c = old; \ - \ - smp_mb(); \ - \ - c; \ -}) - -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - -#define atomic_inc(v) atomic_add(1, v) -#define atomic_dec(v) atomic_sub(1, v) - -#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) -#define atomic_inc_return(v) atomic_add_return(1, (v)) -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0) - - #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> @@ -472,7 +429,8 @@ static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \ ATOMIC64_OP_RETURN(op, op1, op2) \ ATOMIC64_FETCH_OP(op, op1, op2) -#define atomic64_andnot atomic64_andnot +#define atomic64_andnot atomic64_andnot +#define atomic64_fetch_andnot atomic64_fetch_andnot ATOMIC64_OPS(add, add.f, adc) ATOMIC64_OPS(sub, sub.f, sbc) @@ -559,53 +517,43 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) return val; } +#define atomic64_dec_if_positive atomic64_dec_if_positive /** - * atomic64_add_unless - add unless the number is a given value + * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. * - * if (v != u) { v += a; ret = 1} else {ret = 0} - * Returns 1 iff @v was not @u (i.e. if add actually happened) + * Atomically adds @a to @v, if it was not @u. + * Returns the old value of @v */ -static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) +static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, + long long u) { - long long val; - int op_done; + long long old, temp; smp_mb(); __asm__ __volatile__( "1: llockd %0, [%2] \n" - " mov %1, 1 \n" " brne %L0, %L4, 2f # continue to add since v != u \n" " breq.d %H0, %H4, 3f # return since v == u \n" - " mov %1, 0 \n" "2: \n" - " add.f %L0, %L0, %L3 \n" - " adc %H0, %H0, %H3 \n" - " scondd %0, [%2] \n" + " add.f %L1, %L0, %L3 \n" + " adc %H1, %H0, %H3 \n" + " scondd %1, [%2] \n" " bnz 1b \n" "3: \n" - : "=&r"(val), "=&r" (op_done) + : "=&r"(old), "=&r" (temp) : "r"(&v->counter), "r"(a), "r"(u) : "cc"); /* memory clobber comes from smp_mb() */ smp_mb(); - return op_done; + return old; } - -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) -#define atomic64_inc(v) atomic64_add(1LL, (v)) -#define atomic64_inc_return(v) atomic64_add_return(1LL, (v)) -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec(v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v)) -#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) +#define atomic64_fetch_add_unless atomic64_fetch_add_unless #endif /* !CONFIG_GENERIC_ATOMIC64 */ diff --git a/arch/arc/include/asm/cache.h b/arch/arc/include/asm/cache.h index 8486f328cc5d..ff7d3232764a 100644 --- a/arch/arc/include/asm/cache.h +++ b/arch/arc/include/asm/cache.h @@ -48,7 +48,9 @@ }) /* Largest line length for either L1 or L2 is 128 bytes */ -#define ARCH_DMA_MINALIGN 128 +#define SMP_CACHE_BYTES 128 +#define cache_line_size() SMP_CACHE_BYTES +#define ARCH_DMA_MINALIGN SMP_CACHE_BYTES extern void arc_cache_init(void); extern char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len); diff --git a/arch/arc/include/asm/delay.h b/arch/arc/include/asm/delay.h index d5da2115d78a..03d6bb0f4e13 100644 --- a/arch/arc/include/asm/delay.h +++ b/arch/arc/include/asm/delay.h @@ -17,8 +17,11 @@ #ifndef __ASM_ARC_UDELAY_H #define __ASM_ARC_UDELAY_H +#include <asm-generic/types.h> #include <asm/param.h> /* HZ */ +extern unsigned long loops_per_jiffy; + static inline void __delay(unsigned long loops) { __asm__ __volatile__( diff --git a/arch/arc/include/asm/entry-compact.h b/arch/arc/include/asm/entry-compact.h index ec36d5b6d435..29f3988c9424 100644 --- a/arch/arc/include/asm/entry-compact.h +++ b/arch/arc/include/asm/entry-compact.h @@ -234,6 +234,9 @@ POP gp RESTORE_R12_TO_R0 +#ifdef CONFIG_ARC_CURR_IN_REG + ld r25, [sp, 12] +#endif ld sp, [sp] /* restore original sp */ /* orig_r0, ECR, user_r25 skipped automatically */ .endm @@ -315,6 +318,9 @@ POP gp RESTORE_R12_TO_R0 +#ifdef CONFIG_ARC_CURR_IN_REG + ld r25, [sp, 12] +#endif ld sp, [sp] /* restore original sp */ /* orig_r0, ECR, user_r25 skipped automatically */ .endm diff --git a/arch/arc/include/asm/entry.h b/arch/arc/include/asm/entry.h index 51597f344a62..302b0db8ea2b 100644 --- a/arch/arc/include/asm/entry.h +++ b/arch/arc/include/asm/entry.h @@ -86,9 +86,6 @@ POP r1 POP r0 -#ifdef CONFIG_ARC_CURR_IN_REG - ld r25, [sp, 12] -#endif .endm /*-------------------------------------------------------------- diff --git a/arch/arc/include/asm/kprobes.h b/arch/arc/include/asm/kprobes.h index 2e52d18e6bc7..2c1b479d5aea 100644 --- a/arch/arc/include/asm/kprobes.h +++ b/arch/arc/include/asm/kprobes.h @@ -45,8 +45,6 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned int kprobe_status; - struct pt_regs jprobe_saved_regs; - char jprobes_stack[MAX_STACK_SIZE]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/arc/include/asm/mach_desc.h b/arch/arc/include/asm/mach_desc.h index c28e6c347b49..871f3cb16af9 100644 --- a/arch/arc/include/asm/mach_desc.h +++ b/arch/arc/include/asm/mach_desc.h @@ -34,9 +34,7 @@ struct machine_desc { const char *name; const char **dt_compat; void (*init_early)(void); -#ifdef CONFIG_SMP void (*init_per_cpu)(unsigned int); -#endif void (*init_machine)(void); void (*init_late)(void); diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h index 109baa06831c..09ddddf71cc5 100644 --- a/arch/arc/include/asm/page.h +++ b/arch/arc/include/asm/page.h @@ -105,7 +105,7 @@ typedef pte_t * pgtable_t; #define virt_addr_valid(kaddr) pfn_valid(virt_to_pfn(kaddr)) /* Default Permissions for stack/heaps pages (Non Executable) */ -#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE) +#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) #define WANT_PAGE_VIRTUAL 1 diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h index 8ec5599a0957..cf4be70d5892 100644 --- a/arch/arc/include/asm/pgtable.h +++ b/arch/arc/include/asm/pgtable.h @@ -377,7 +377,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, /* Decode a PTE containing swap "identifier "into constituents */ #define __swp_type(pte_lookalike) (((pte_lookalike).val) & 0x1f) -#define __swp_offset(pte_lookalike) ((pte_lookalike).val << 13) +#define __swp_offset(pte_lookalike) ((pte_lookalike).val >> 13) /* NOPs, to keep generic kernel happy */ #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) diff --git a/arch/arc/kernel/irq.c b/arch/arc/kernel/irq.c index 538b36afe89e..62b185057c04 100644 --- a/arch/arc/kernel/irq.c +++ b/arch/arc/kernel/irq.c @@ -31,10 +31,10 @@ void __init init_IRQ(void) /* a SMP H/w block could do IPI IRQ request here */ if (plat_smp_ops.init_per_cpu) plat_smp_ops.init_per_cpu(smp_processor_id()); +#endif if (machine_desc->init_per_cpu) machine_desc->init_per_cpu(smp_processor_id()); -#endif } /* diff --git a/arch/arc/kernel/kprobes.c b/arch/arc/kernel/kprobes.c index 42b05046fad9..df35d4c0b0b8 100644 --- a/arch/arc/kernel/kprobes.c +++ b/arch/arc/kernel/kprobes.c @@ -225,24 +225,18 @@ int __kprobes arc_kprobe_handler(unsigned long addr, struct pt_regs *regs) /* If we have no pre-handler or it returned 0, we continue with * normal processing. If we have a pre-handler and it returned - * non-zero - which is expected from setjmp_pre_handler for - * jprobe, we return without single stepping and leave that to - * the break-handler which is invoked by a kprobe from - * jprobe_return + * non-zero - which means user handler setup registers to exit + * to another instruction, we must skip the single stepping. */ if (!p->pre_handler || !p->pre_handler(p, regs)) { setup_singlestep(p, regs); kcb->kprobe_status = KPROBE_HIT_SS; + } else { + reset_current_kprobe(); + preempt_enable_no_resched(); } return 1; - } else if (kprobe_running()) { - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - setup_singlestep(p, regs); - kcb->kprobe_status = KPROBE_HIT_SS; - return 1; - } } /* no_kprobe: */ @@ -386,38 +380,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, return ret; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long sp_addr = regs->sp; - - kcb->jprobe_saved_regs = *regs; - memcpy(kcb->jprobes_stack, (void *)sp_addr, MIN_STACK_SIZE(sp_addr)); - regs->ret = (unsigned long)(jp->entry); - - return 1; -} - -void __kprobes jprobe_return(void) -{ - __asm__ __volatile__("unimp_s"); - return; -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long sp_addr; - - *regs = kcb->jprobe_saved_regs; - sp_addr = regs->sp; - memcpy((void *)sp_addr, kcb->jprobes_stack, MIN_STACK_SIZE(sp_addr)); - preempt_enable_no_resched(); - - return 1; -} - static void __used kretprobe_trampoline_holder(void) { __asm__ __volatile__(".global kretprobe_trampoline\n" @@ -483,9 +445,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, kretprobe_assert(ri, orig_ret_address, trampoline_address); regs->ret = orig_ret_address; - reset_current_kprobe(); kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 5ac3b547453f..4674541eba3f 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -47,7 +47,8 @@ SYSCALL_DEFINE0(arc_gettls) SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) { struct pt_regs *regs = current_pt_regs(); - int uval = -EFAULT; + u32 uval; + int ret; /* * This is only for old cores lacking LLOCK/SCOND, which by defintion @@ -60,23 +61,47 @@ SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) /* Z indicates to userspace if operation succeded */ regs->status32 &= ~STATUS_Z_MASK; - if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) - return -EFAULT; + ret = access_ok(VERIFY_WRITE, uaddr, sizeof(*uaddr)); + if (!ret) + goto fail; +again: preempt_disable(); - if (__get_user(uval, uaddr)) - goto done; + ret = __get_user(uval, uaddr); + if (ret) + goto fault; - if (uval == expected) { - if (!__put_user(new, uaddr)) - regs->status32 |= STATUS_Z_MASK; - } + if (uval != expected) + goto out; -done: - preempt_enable(); + ret = __put_user(new, uaddr); + if (ret) + goto fault; + + regs->status32 |= STATUS_Z_MASK; +out: + preempt_enable(); return uval; + +fault: + preempt_enable(); + + if (unlikely(ret != -EFAULT)) + goto fail; + + down_read(¤t->mm->mmap_sem); + ret = fixup_user_fault(current, current->mm, (unsigned long) uaddr, + FAULT_FLAG_WRITE, NULL); + up_read(¤t->mm->mmap_sem); + + if (likely(!ret)) + goto again; + +fail: + force_sig(SIGSEGV, current); + return ret; } #ifdef CONFIG_ISA_ARCV2 diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c index 9dbe645ee127..25c631942500 100644 --- a/arch/arc/mm/cache.c +++ b/arch/arc/mm/cache.c @@ -1038,7 +1038,7 @@ void flush_cache_mm(struct mm_struct *mm) void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, unsigned long pfn) { - unsigned int paddr = pfn << PAGE_SHIFT; + phys_addr_t paddr = pfn << PAGE_SHIFT; u_vaddr &= PAGE_MASK; @@ -1058,8 +1058,9 @@ void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long u_vaddr) { /* TBD: do we really need to clear the kernel mapping */ - __flush_dcache_page(page_address(page), u_vaddr); - __flush_dcache_page(page_address(page), page_address(page)); + __flush_dcache_page((phys_addr_t)page_address(page), u_vaddr); + __flush_dcache_page((phys_addr_t)page_address(page), + (phys_addr_t)page_address(page)); } @@ -1246,6 +1247,16 @@ void __init arc_cache_init_master(void) } } + /* + * Check that SMP_CACHE_BYTES (and hence ARCH_DMA_MINALIGN) is larger + * or equal to any cache line length. + */ + BUILD_BUG_ON_MSG(L1_CACHE_BYTES > SMP_CACHE_BYTES, + "SMP_CACHE_BYTES must be >= any cache line length"); + if (is_isa_arcv2() && (l2_line_sz > SMP_CACHE_BYTES)) + panic("L2 Cache line [%d] > kernel Config [%d]\n", + l2_line_sz, SMP_CACHE_BYTES); + /* Note that SLC disable not formally supported till HS 3.0 */ if (is_isa_arcv2() && l2_line_sz && !slc_enable) arc_slc_disable(); diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 8c1071840979..ec47e6079f5d 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -129,14 +129,59 @@ int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, return ret; } +/* + * Cache operations depending on function and direction argument, inspired by + * https://lkml.org/lkml/2018/5/18/979 + * "dma_sync_*_for_cpu and direction=TO_DEVICE (was Re: [PATCH 02/20] + * dma-mapping: provide a generic dma-noncoherent implementation)" + * + * | map == for_device | unmap == for_cpu + * |---------------------------------------------------------------- + * TO_DEV | writeback writeback | none none + * FROM_DEV | invalidate invalidate | invalidate* invalidate* + * BIDIR | writeback+inv writeback+inv | invalidate invalidate + * + * [*] needed for CPU speculative prefetches + * + * NOTE: we don't check the validity of direction argument as it is done in + * upper layer functions (in include/linux/dma-mapping.h) + */ + void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - dma_cache_wback(paddr, size); + switch (dir) { + case DMA_TO_DEVICE: + dma_cache_wback(paddr, size); + break; + + case DMA_FROM_DEVICE: + dma_cache_inv(paddr, size); + break; + + case DMA_BIDIRECTIONAL: + dma_cache_wback_inv(paddr, size); + break; + + default: + break; + } } void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, size_t size, enum dma_data_direction dir) { - dma_cache_inv(paddr, size); + switch (dir) { + case DMA_TO_DEVICE: + break; + + /* FROM_DEVICE invalidate needed if speculative CPU prefetch only */ + case DMA_FROM_DEVICE: + case DMA_BIDIRECTIONAL: + dma_cache_inv(paddr, size); + break; + + default: + break; + } } diff --git a/arch/arc/plat-eznps/include/plat/ctop.h b/arch/arc/plat-eznps/include/plat/ctop.h index 0c7d11022d0f..4f6a1673b3a6 100644 --- a/arch/arc/plat-eznps/include/plat/ctop.h +++ b/arch/arc/plat-eznps/include/plat/ctop.h @@ -21,6 +21,7 @@ #error "Incorrect ctop.h include" #endif +#include <linux/types.h> #include <soc/nps/common.h> /* core auxiliary registers */ @@ -143,6 +144,15 @@ struct nps_host_reg_gim_p_int_dst { }; /* AUX registers definition */ +struct nps_host_reg_aux_dpc { + union { + struct { + u32 ien:1, men:1, hen:1, reserved:29; + }; + u32 value; + }; +}; + struct nps_host_reg_aux_udmc { union { struct { diff --git a/arch/arc/plat-eznps/mtm.c b/arch/arc/plat-eznps/mtm.c index 2388de3d09ef..ed0077ef666e 100644 --- a/arch/arc/plat-eznps/mtm.c +++ b/arch/arc/plat-eznps/mtm.c @@ -15,6 +15,8 @@ */ #include <linux/smp.h> +#include <linux/init.h> +#include <linux/kernel.h> #include <linux/io.h> #include <linux/log2.h> #include <asm/arcregs.h> @@ -157,10 +159,10 @@ void mtm_enable_core(unsigned int cpu) /* Verify and set the value of the mtm hs counter */ static int __init set_mtm_hs_ctr(char *ctr_str) { - long hs_ctr; + int hs_ctr; int ret; - ret = kstrtol(ctr_str, 0, &hs_ctr); + ret = kstrtoint(ctr_str, 0, &hs_ctr); if (ret || hs_ctr > MT_HS_CNT_MAX || hs_ctr < MT_HS_CNT_MIN) { pr_err("** Invalid @nps_mtm_hs_ctr [%d] needs to be [%d:%d] (incl)\n", diff --git a/arch/arc/plat-hsdk/Kconfig b/arch/arc/plat-hsdk/Kconfig index 19ab3cf98f0f..9356753c2ed8 100644 --- a/arch/arc/plat-hsdk/Kconfig +++ b/arch/arc/plat-hsdk/Kconfig @@ -7,5 +7,8 @@ menuconfig ARC_SOC_HSDK bool "ARC HS Development Kit SOC" + depends on ISA_ARCV2 + select ARC_HAS_ACCL_REGS select CLK_HSDK select RESET_HSDK + select MIGHT_HAVE_PCI diff --git a/arch/arc/plat-hsdk/platform.c b/arch/arc/plat-hsdk/platform.c index 2958aedb649a..2588b842407c 100644 --- a/arch/arc/plat-hsdk/platform.c +++ b/arch/arc/plat-hsdk/platform.c @@ -42,6 +42,66 @@ static void __init hsdk_init_per_cpu(unsigned int cpu) #define SDIO_UHS_REG_EXT (SDIO_BASE + 0x108) #define SDIO_UHS_REG_EXT_DIV_2 (2 << 30) +#define HSDK_GPIO_INTC (ARC_PERIPHERAL_BASE + 0x3000) + +static void __init hsdk_enable_gpio_intc_wire(void) +{ + /* + * Peripherals on CPU Card are wired to cpu intc via intermediate + * DW APB GPIO blocks (mainly for debouncing) + * + * --------------------- + * | snps,archs-intc | + * --------------------- + * | + * ---------------------- + * | snps,archs-idu-intc | + * ---------------------- + * | | | | | + * | [eth] [USB] [... other peripherals] + * | + * ------------------- + * | snps,dw-apb-intc | + * ------------------- + * | | | | + * [Bt] [HAPS] [... other peripherals] + * + * Current implementation of "irq-dw-apb-ictl" driver doesn't work well + * with stacked INTCs. In particular problem happens if its master INTC + * not yet instantiated. See discussion here - + * https://lkml.org/lkml/2015/3/4/755 + * + * So setup the first gpio block as a passive pass thru and hide it from + * DT hardware topology - connect intc directly to cpu intc + * The GPIO "wire" needs to be init nevertheless (here) + * + * One side adv is that peripheral interrupt handling avoids one nested + * intc ISR hop + * + * According to HSDK User's Manual [1], "Table 2 Interrupt Mapping" + * we have the following GPIO input lines used as sources of interrupt: + * - GPIO[0] - Bluetooth interrupt of RS9113 module + * - GPIO[2] - HAPS interrupt (on HapsTrak 3 connector) + * - GPIO[3] - Audio codec (MAX9880A) interrupt + * - GPIO[8-23] - Available on Arduino and PMOD_x headers + * For now there's no use of Arduino and PMOD_x headers in Linux + * use-case so we only enable lines 0, 2 and 3. + * + * [1] https://github.com/foss-for-synopsys-dwc-arc-processors/ARC-Development-Systems-Forum/wiki/docs/ARC_HSDK_User_Guide.pdf + */ +#define GPIO_INTEN (HSDK_GPIO_INTC + 0x30) +#define GPIO_INTMASK (HSDK_GPIO_INTC + 0x34) +#define GPIO_INTTYPE_LEVEL (HSDK_GPIO_INTC + 0x38) +#define GPIO_INT_POLARITY (HSDK_GPIO_INTC + 0x3c) +#define GPIO_INT_CONNECTED_MASK 0x0d + + iowrite32(0xffffffff, (void __iomem *) GPIO_INTMASK); + iowrite32(~GPIO_INT_CONNECTED_MASK, (void __iomem *) GPIO_INTMASK); + iowrite32(0x00000000, (void __iomem *) GPIO_INTTYPE_LEVEL); + iowrite32(0xffffffff, (void __iomem *) GPIO_INT_POLARITY); + iowrite32(GPIO_INT_CONNECTED_MASK, (void __iomem *) GPIO_INTEN); +} + static void __init hsdk_init_early(void) { /* @@ -62,6 +122,8 @@ static void __init hsdk_init_early(void) * minimum possible div-by-2. */ iowrite32(SDIO_UHS_REG_EXT_DIV_2, (void __iomem *) SDIO_UHS_REG_EXT); + + hsdk_enable_gpio_intc_wire(); } static const char *hsdk_compat[] __initconst = { diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 843edfd000be..0f328d639d51 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -9,6 +9,7 @@ config ARM select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_KCOV + select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PTE_SPECIAL if ARM_LPAE select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_SET_MEMORY @@ -337,8 +338,8 @@ config ARCH_MULTIPLATFORM select TIMER_OF select COMMON_CLK select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_MULTI_HANDLER select MIGHT_HAVE_PCI - select MULTI_IRQ_HANDLER select PCI_DOMAINS if PCI select SPARSE_IRQ select USE_OF @@ -465,9 +466,9 @@ config ARCH_DOVE bool "Marvell Dove" select CPU_PJ4 select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select MIGHT_HAVE_PCI - select MULTI_IRQ_HANDLER select MVEBU_MBUS select PINCTRL select PINCTRL_DOVE @@ -512,8 +513,8 @@ config ARCH_LPC32XX select COMMON_CLK select CPU_ARM926T select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB - select MULTI_IRQ_HANDLER select SPARSE_IRQ select USE_OF help @@ -532,11 +533,11 @@ config ARCH_PXA select TIMER_OF select CPU_XSCALE if !CPU_XSC3 select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_MULTI_HANDLER select GPIO_PXA select GPIOLIB select HAVE_IDE select IRQ_DOMAIN - select MULTI_IRQ_HANDLER select PLAT_PXA select SPARSE_IRQ help @@ -572,11 +573,11 @@ config ARCH_SA1100 select CPU_FREQ select CPU_SA1100 select GENERIC_CLOCKEVENTS + select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_IDE select IRQ_DOMAIN select ISA - select MULTI_IRQ_HANDLER select NEED_MACH_MEMORY_H select SPARSE_IRQ help @@ -590,10 +591,10 @@ config ARCH_S3C24XX select GENERIC_CLOCKEVENTS select GPIO_SAMSUNG select GPIOLIB + select GENERIC_IRQ_MULTI_HANDLER select HAVE_S3C2410_I2C if I2C select HAVE_S3C2410_WATCHDOG if WATCHDOG select HAVE_S3C_RTC if RTC_CLASS - select MULTI_IRQ_HANDLER select NEED_MACH_IO_H select SAMSUNG_ATAGS select USE_OF @@ -627,10 +628,10 @@ config ARCH_OMAP1 select CLKSRC_MMIO select GENERIC_CLOCKEVENTS select GENERIC_IRQ_CHIP + select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_IDE select IRQ_DOMAIN - select MULTI_IRQ_HANDLER select NEED_MACH_IO_H if PCCARD select NEED_MACH_MEMORY_H select SPARSE_IRQ @@ -921,11 +922,6 @@ config IWMMXT Enable support for iWMMXt context switching at run time if running on a CPU that supports it. -config MULTI_IRQ_HANDLER - bool - help - Allow each machine to specify it's own IRQ handler at run time. - if !MMU source "arch/arm/Kconfig-nommu" endif diff --git a/arch/arm/Makefile b/arch/arm/Makefile index fc26c3d7b9b6..62ebeae9f837 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -46,12 +46,12 @@ ifeq ($(CONFIG_CPU_BIG_ENDIAN),y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__ARMEB__ AS += -EB -LD += -EB +LDFLAGS += -EB else KBUILD_CPPFLAGS += -mlittle-endian CHECKFLAGS += -D__ARMEL__ AS += -EL -LD += -EL +LDFLAGS += -EL endif # diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi index f9e8667f5886..73b514dddf65 100644 --- a/arch/arm/boot/dts/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/am335x-bone-common.dtsi @@ -168,7 +168,6 @@ AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat3.mmc0_dat3 */ AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_cmd.mmc0_cmd */ AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_clk.mmc0_clk */ - AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4) /* mcasp0_aclkr.mmc0_sdwp */ >; }; diff --git a/arch/arm/boot/dts/am3517.dtsi b/arch/arm/boot/dts/am3517.dtsi index ca294914bbb1..23ea381d363f 100644 --- a/arch/arm/boot/dts/am3517.dtsi +++ b/arch/arm/boot/dts/am3517.dtsi @@ -39,6 +39,8 @@ ti,davinci-ctrl-ram-size = <0x2000>; ti,davinci-rmii-en = /bits/ 8 <1>; local-mac-address = [ 00 00 00 00 00 00 ]; + clocks = <&emac_ick>; + clock-names = "ick"; }; davinci_mdio: ethernet@5c030000 { @@ -49,6 +51,8 @@ bus_freq = <1000000>; #address-cells = <1>; #size-cells = <0>; + clocks = <&emac_fck>; + clock-names = "fck"; }; uart4: serial@4809e000 { @@ -87,6 +91,11 @@ }; }; +/* Table Table 5-79 of the TRM shows 480ab000 is reserved */ +&usb_otg_hs { + status = "disabled"; +}; + &iva { status = "disabled"; }; diff --git a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts index 440351ad0b80..d4be3fd0b6f4 100644 --- a/arch/arm/boot/dts/am437x-sk-evm.dts +++ b/arch/arm/boot/dts/am437x-sk-evm.dts @@ -610,6 +610,8 @@ touchscreen-size-x = <480>; touchscreen-size-y = <272>; + + wakeup-source; }; tlv320aic3106: tlv320aic3106@1b { diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi index 18edc9bc7927..929459c42760 100644 --- a/arch/arm/boot/dts/armada-38x.dtsi +++ b/arch/arm/boot/dts/armada-38x.dtsi @@ -547,7 +547,7 @@ thermal: thermal@e8078 { compatible = "marvell,armada380-thermal"; - reg = <0xe4078 0x4>, <0xe4074 0x4>; + reg = <0xe4078 0x4>, <0xe4070 0x8>; status = "okay"; }; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 9dcd14edc202..e03495a799ce 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -1580,7 +1580,6 @@ dr_mode = "otg"; snps,dis_u3_susphy_quirk; snps,dis_u2_susphy_quirk; - snps,dis_metastability_quirk; }; }; @@ -1608,6 +1607,7 @@ dr_mode = "otg"; snps,dis_u3_susphy_quirk; snps,dis_u2_susphy_quirk; + snps,dis_metastability_quirk; }; }; diff --git a/arch/arm/boot/dts/imx51-zii-rdu1.dts b/arch/arm/boot/dts/imx51-zii-rdu1.dts index df9eca94d812..8a878687197b 100644 --- a/arch/arm/boot/dts/imx51-zii-rdu1.dts +++ b/arch/arm/boot/dts/imx51-zii-rdu1.dts @@ -770,7 +770,7 @@ pinctrl_ts: tsgrp { fsl,pins = < - MX51_PAD_CSI1_D8__GPIO3_12 0x85 + MX51_PAD_CSI1_D8__GPIO3_12 0x04 MX51_PAD_CSI1_D9__GPIO3_13 0x85 >; }; diff --git a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi index 19a075aee19e..f14df0baf2ab 100644 --- a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi +++ b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi @@ -692,7 +692,7 @@ dsa,member = <0 0>; eeprom-length = <512>; interrupt-parent = <&gpio6>; - interrupts = <3 IRQ_TYPE_EDGE_FALLING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; interrupt-controller; #interrupt-cells = <2>; diff --git a/arch/arm/boot/dts/omap4-droid4-xt894.dts b/arch/arm/boot/dts/omap4-droid4-xt894.dts index bdf73cbcec3a..e7c3c563ff8f 100644 --- a/arch/arm/boot/dts/omap4-droid4-xt894.dts +++ b/arch/arm/boot/dts/omap4-droid4-xt894.dts @@ -159,13 +159,7 @@ dais = <&mcbsp2_port>, <&mcbsp3_port>; }; -}; - -&dss { - status = "okay"; -}; -&gpio6 { pwm8: dmtimer-pwm-8 { pinctrl-names = "default"; pinctrl-0 = <&vibrator_direction_pin>; @@ -192,7 +186,10 @@ pwm-names = "enable", "direction"; direction-duty-cycle-ns = <10000000>; }; +}; +&dss { + status = "okay"; }; &dsi1 { diff --git a/arch/arm/configs/imx_v4_v5_defconfig b/arch/arm/configs/imx_v4_v5_defconfig index 054591dc9a00..4cd2f4a2bff4 100644 --- a/arch/arm/configs/imx_v4_v5_defconfig +++ b/arch/arm/configs/imx_v4_v5_defconfig @@ -141,9 +141,11 @@ CONFIG_USB_STORAGE=y CONFIG_USB_CHIPIDEA=y CONFIG_USB_CHIPIDEA_UDC=y CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_CHIPIDEA_ULPI=y CONFIG_NOP_USB_XCEIV=y CONFIG_USB_GADGET=y CONFIG_USB_ETH=m +CONFIG_USB_ULPI_BUS=y CONFIG_MMC=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index f70507ab91ee..200ebda47e0c 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -302,6 +302,7 @@ CONFIG_USB_STORAGE=y CONFIG_USB_CHIPIDEA=y CONFIG_USB_CHIPIDEA_UDC=y CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_CHIPIDEA_ULPI=y CONFIG_USB_SERIAL=m CONFIG_USB_SERIAL_GENERIC=y CONFIG_USB_SERIAL_FTDI_SIO=m @@ -338,6 +339,7 @@ CONFIG_USB_GADGETFS=m CONFIG_USB_FUNCTIONFS=m CONFIG_USB_MASS_STORAGE=m CONFIG_USB_G_SERIAL=m +CONFIG_USB_ULPI_BUS=y CONFIG_MMC=y CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_PLTFM=y diff --git a/arch/arm/crypto/speck-neon-core.S b/arch/arm/crypto/speck-neon-core.S index 3c1e203e53b9..57caa742016e 100644 --- a/arch/arm/crypto/speck-neon-core.S +++ b/arch/arm/crypto/speck-neon-core.S @@ -272,9 +272,11 @@ * Allocate stack space to store 128 bytes worth of tweaks. For * performance, this space is aligned to a 16-byte boundary so that we * can use the load/store instructions that declare 16-byte alignment. + * For Thumb2 compatibility, don't do the 'bic' directly on 'sp'. */ - sub sp, #128 - bic sp, #0xf + sub r12, sp, #128 + bic r12, #0xf + mov sp, r12 .if \n == 64 // Load first tweak diff --git a/arch/arm/firmware/Makefile b/arch/arm/firmware/Makefile index a71f16536b6c..6e41336b0bc4 100644 --- a/arch/arm/firmware/Makefile +++ b/arch/arm/firmware/Makefile @@ -1 +1,4 @@ obj-$(CONFIG_TRUSTED_FOUNDATIONS) += trusted_foundations.o + +# tf_generic_smc() fails to build with -fsanitize-coverage=trace-pc +KCOV_INSTRUMENT := n diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 0cd4dccbae78..b17ee03d280b 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -460,6 +460,10 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) adds \tmp, \addr, #\size - 1 sbcccs \tmp, \tmp, \limit bcs \bad +#ifdef CONFIG_CPU_SPECTRE + movcs \addr, #0 + csdb +#endif #endif .endm diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 66d0e215a773..f74756641410 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -130,7 +130,7 @@ static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) } #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int oldval, newval; unsigned long tmp; @@ -156,6 +156,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return oldval; } +#define atomic_fetch_add_unless atomic_fetch_add_unless #else /* ARM_ARCH_6 */ @@ -215,15 +216,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - - c = atomic_read(v); - while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c) - c = old; - return c; -} +#define atomic_fetch_andnot atomic_fetch_andnot #endif /* __LINUX_ARM_ARCH__ */ @@ -254,17 +247,6 @@ ATOMIC_OPS(xor, ^=, eor) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -#define atomic_inc(v) atomic_add(1, v) -#define atomic_dec(v) atomic_sub(1, v) - -#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) -#define atomic_inc_return_relaxed(v) (atomic_add_return_relaxed(1, v)) -#define atomic_dec_return_relaxed(v) (atomic_sub_return_relaxed(1, v)) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0) - #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct { long long counter; @@ -494,12 +476,13 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) return result; } +#define atomic64_dec_if_positive atomic64_dec_if_positive -static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) +static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, + long long u) { - long long val; + long long oldval, newval; unsigned long tmp; - int ret = 1; smp_mb(); prefetchw(&v->counter); @@ -508,33 +491,23 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) "1: ldrexd %0, %H0, [%4]\n" " teq %0, %5\n" " teqeq %H0, %H5\n" -" moveq %1, #0\n" " beq 2f\n" -" adds %Q0, %Q0, %Q6\n" -" adc %R0, %R0, %R6\n" -" strexd %2, %0, %H0, [%4]\n" +" adds %Q1, %Q0, %Q6\n" +" adc %R1, %R0, %R6\n" +" strexd %2, %1, %H1, [%4]\n" " teq %2, #0\n" " bne 1b\n" "2:" - : "=&r" (val), "+r" (ret), "=&r" (tmp), "+Qo" (v->counter) + : "=&r" (oldval), "=&r" (newval), "=&r" (tmp), "+Qo" (v->counter) : "r" (&v->counter), "r" (u), "r" (a) : "cc"); - if (ret) + if (oldval != u) smp_mb(); - return ret; + return oldval; } - -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) -#define atomic64_inc(v) atomic64_add(1LL, (v)) -#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1LL, (v)) -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec(v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1LL, (v)) -#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) +#define atomic64_fetch_add_unless atomic64_fetch_add_unless #endif /* !CONFIG_GENERIC_ATOMIC64 */ #endif diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 4cab9bb823fb..c92e42a5c8f7 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -215,7 +215,6 @@ extern int _find_next_bit_be(const unsigned long *p, int size, int offset); #if __LINUX_ARM_ARCH__ < 5 -#include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/fls.h> @@ -223,93 +222,20 @@ extern int _find_next_bit_be(const unsigned long *p, int size, int offset); #else -static inline int constant_fls(int x) -{ - int r = 32; - - if (!x) - return 0; - if (!(x & 0xffff0000u)) { - x <<= 16; - r -= 16; - } - if (!(x & 0xff000000u)) { - x <<= 8; - r -= 8; - } - if (!(x & 0xf0000000u)) { - x <<= 4; - r -= 4; - } - if (!(x & 0xc0000000u)) { - x <<= 2; - r -= 2; - } - if (!(x & 0x80000000u)) { - x <<= 1; - r -= 1; - } - return r; -} - -/* - * On ARMv5 and above those functions can be implemented around the - * clz instruction for much better code efficiency. __clz returns - * the number of leading zeros, zero input will return 32, and - * 0x80000000 will return 0. - */ -static inline unsigned int __clz(unsigned int x) -{ - unsigned int ret; - - asm("clz\t%0, %1" : "=r" (ret) : "r" (x)); - - return ret; -} - -/* - * fls() returns zero if the input is zero, otherwise returns the bit - * position of the last set bit, where the LSB is 1 and MSB is 32. - */ -static inline int fls(int x) -{ - if (__builtin_constant_p(x)) - return constant_fls(x); - - return 32 - __clz(x); -} - -/* - * __fls() returns the bit position of the last bit set, where the - * LSB is 0 and MSB is 31. Zero input is undefined. - */ -static inline unsigned long __fls(unsigned long x) -{ - return fls(x) - 1; -} - -/* - * ffs() returns zero if the input was zero, otherwise returns the bit - * position of the first set bit, where the LSB is 1 and MSB is 32. - */ -static inline int ffs(int x) -{ - return fls(x & -x); -} - /* - * __ffs() returns the bit position of the first bit set, where the - * LSB is 0 and MSB is 31. Zero input is undefined. + * On ARMv5 and above, the gcc built-ins may rely on the clz instruction + * and produce optimal inlined code in all cases. On ARMv7 it is even + * better by also using the rbit instruction. */ -static inline unsigned long __ffs(unsigned long x) -{ - return ffs(x) - 1; -} - -#define ffz(x) __ffs( ~(x) ) +#include <asm-generic/bitops/builtin-__fls.h> +#include <asm-generic/bitops/builtin-__ffs.h> +#include <asm-generic/bitops/builtin-fls.h> +#include <asm-generic/bitops/builtin-ffs.h> #endif +#include <asm-generic/bitops/ffz.h> + #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/sched.h> diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 17f1f1a814ff..38badaae8d9d 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -58,6 +58,9 @@ void efi_virtmap_unload(void); #define efi_call_runtime(f, ...) sys_table_arg->runtime->f(__VA_ARGS__) #define efi_is_64bit() (false) +#define efi_table_attr(table, attr, instance) \ + ((table##_t *)instance)->attr + #define efi_call_proto(protocol, f, instance, ...) \ ((protocol##_t *)instance)->f(instance, ##__VA_ARGS__) diff --git a/arch/arm/include/asm/hw_breakpoint.h b/arch/arm/include/asm/hw_breakpoint.h index e46e4e7bdba3..ac54c06764e6 100644 --- a/arch/arm/include/asm/hw_breakpoint.h +++ b/arch/arm/include/asm/hw_breakpoint.h @@ -111,14 +111,17 @@ static inline void decode_ctrl_reg(u32 reg, asm volatile("mcr p14, 0, %0, " #N "," #M ", " #OP2 : : "r" (VAL));\ } while (0) +struct perf_event_attr; struct notifier_block; struct perf_event; struct pmu; extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, int *gen_len, int *gen_type); -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index b6f319606e30..c883fcbe93b6 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -31,11 +31,6 @@ extern void asm_do_IRQ(unsigned int, struct pt_regs *); void handle_IRQ(unsigned int, struct pt_regs *); void init_IRQ(void); -#ifdef CONFIG_MULTI_IRQ_HANDLER -extern void (*handle_arch_irq)(struct pt_regs *); -extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); -#endif - #ifdef CONFIG_SMP extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self); diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index 59655459da59..82290f212d8e 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -44,8 +44,6 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned int kprobe_status; struct prev_kprobe prev_kprobe; - struct pt_regs jprobe_saved_regs; - char jprobes_stack[MAX_STACK_SIZE]; }; void arch_remove_kprobe(struct kprobe *); diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 5c1ad11aa392..bb8851208e17 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -59,7 +59,7 @@ struct machine_desc { void (*init_time)(void); void (*init_machine)(void); void (*init_late)(void); -#ifdef CONFIG_MULTI_IRQ_HANDLER +#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER void (*handle_irq)(struct pt_regs *); #endif void (*restart)(enum reboot_mode, const char *); diff --git a/arch/arm/include/asm/mach/time.h b/arch/arm/include/asm/mach/time.h index 0f79e4dec7f9..4ac3a019a46f 100644 --- a/arch/arm/include/asm/mach/time.h +++ b/arch/arm/include/asm/mach/time.h @@ -13,7 +13,6 @@ extern void timer_tick(void); typedef void (*clock_access_fn)(struct timespec64 *); -extern int register_persistent_clock(clock_access_fn read_boot, - clock_access_fn read_persistent); +extern int register_persistent_clock(clock_access_fn read_persistent); #endif diff --git a/arch/arm/include/asm/probes.h b/arch/arm/include/asm/probes.h index 1e5b9bb92270..991c9127c650 100644 --- a/arch/arm/include/asm/probes.h +++ b/arch/arm/include/asm/probes.h @@ -51,7 +51,6 @@ struct arch_probes_insn { * We assume one instruction can consume at most 64 bytes stack, which is * 'push {r0-r15}'. Instructions consume more or unknown stack space like * 'str r0, [sp, #-80]' and 'str r0, [sp, r1]' should be prohibit to probe. - * Both kprobe and jprobe use this macro. */ #define MAX_STACK_SIZE 64 diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index e71cc35de163..9b37b6ab27fe 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -123,8 +123,8 @@ struct user_vfp_exc; extern int vfp_preserve_user_clear_hwstate(struct user_vfp __user *, struct user_vfp_exc __user *); -extern int vfp_restore_user_hwstate(struct user_vfp __user *, - struct user_vfp_exc __user *); +extern int vfp_restore_user_hwstate(struct user_vfp *, + struct user_vfp_exc *); #endif /* diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index d5562f9ce600..f854148c8d7c 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -292,5 +292,13 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, { } +static inline void tlb_flush_remove_tables(struct mm_struct *mm) +{ +} + +static inline void tlb_flush_remove_tables_local(void *arg) +{ +} + #endif /* CONFIG_MMU */ #endif diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 3d614e90c19f..5451e1f05a19 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -85,6 +85,13 @@ static inline void set_fs(mm_segment_t fs) flag; }) /* + * This is a type: either unsigned long, if the argument fits into + * that type, or otherwise unsigned long long. + */ +#define __inttype(x) \ + __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) + +/* * Single-value transfer routines. They automatically use the right * size if we just have the right pointer type. Note that the functions * which read from user space (*get_*) need to take care not to leak @@ -153,7 +160,7 @@ extern int __get_user_64t_4(void *); ({ \ unsigned long __limit = current_thread_info()->addr_limit - 1; \ register typeof(*(p)) __user *__p asm("r0") = (p); \ - register typeof(x) __r2 asm("r2"); \ + register __inttype(x) __r2 asm("r2"); \ register unsigned long __l asm("r1") = __limit; \ register int __e asm("r0"); \ unsigned int __ua_flags = uaccess_save_and_enable(); \ @@ -243,6 +250,16 @@ static inline void set_fs(mm_segment_t fs) #define user_addr_max() \ (uaccess_kernel() ? ~0UL : get_fs()) +#ifdef CONFIG_CPU_SPECTRE +/* + * When mitigating Spectre variant 1, it is not worth fixing the non- + * verifying accessors, because we need to add verification of the + * address space there. Force these to use the standard get_user() + * version instead. + */ +#define __get_user(x, ptr) get_user(x, ptr) +#else + /* * The "__xxx" versions of the user access functions do not verify the * address space - it must have been done previously with a separate @@ -259,12 +276,6 @@ static inline void set_fs(mm_segment_t fs) __gu_err; \ }) -#define __get_user_error(x, ptr, err) \ -({ \ - __get_user_err((x), (ptr), err); \ - (void) 0; \ -}) - #define __get_user_err(x, ptr, err) \ do { \ unsigned long __gu_addr = (unsigned long)(ptr); \ @@ -324,6 +335,7 @@ do { \ #define __get_user_asm_word(x, addr, err) \ __get_user_asm(x, addr, err, ldr) +#endif #define __put_user_switch(x, ptr, __err, __fn) \ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 179a9f6bd1e3..e85a3af9ddeb 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -22,7 +22,7 @@ #include <asm/glue-df.h> #include <asm/glue-pf.h> #include <asm/vfpmacros.h> -#ifndef CONFIG_MULTI_IRQ_HANDLER +#ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER #include <mach/entry-macro.S> #endif #include <asm/thread_notify.h> @@ -39,7 +39,7 @@ * Interrupt handling. */ .macro irq_handler -#ifdef CONFIG_MULTI_IRQ_HANDLER +#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER ldr r1, =handle_arch_irq mov r0, sp badr lr, 9997f @@ -1226,9 +1226,3 @@ vector_addrexcptn: .globl cr_alignment cr_alignment: .space 4 - -#ifdef CONFIG_MULTI_IRQ_HANDLER - .globl handle_arch_irq -handle_arch_irq: - .space 4 -#endif diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 106a1466518d..746565a876dc 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -48,6 +48,7 @@ saved_pc .req lr * from those features make this path too inefficient. */ ret_fast_syscall: +__ret_fast_syscall: UNWIND(.fnstart ) UNWIND(.cantunwind ) disable_irq_notrace @ disable interrupts @@ -78,6 +79,7 @@ fast_work_pending: * call. */ ret_fast_syscall: +__ret_fast_syscall: UNWIND(.fnstart ) UNWIND(.cantunwind ) str r0, [sp, #S_R0 + S_OFF]! @ save returned r0 @@ -255,7 +257,7 @@ local_restart: tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls? bne __sys_trace - invoke_syscall tbl, scno, r10, ret_fast_syscall + invoke_syscall tbl, scno, r10, __ret_fast_syscall add r1, sp, #S_OFF 2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE) diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index dd546d65a383..ec29de250076 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -53,7 +53,11 @@ ENTRY(stext) THUMB(1: ) #endif - setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode +#ifdef CONFIG_ARM_VIRT_EXT + bl __hyp_stub_install +#endif + @ ensure svc mode and all interrupts masked + safe_svcmode_maskall r9 @ and irqs disabled #if defined(CONFIG_CPU_CP15) mrc p15, 0, r9, c0, c0 @ get processor id @@ -89,7 +93,11 @@ ENTRY(secondary_startup) * the processor type - there is no need to check the machine type * as it has already been validated by the primary processor. */ - setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 +#ifdef CONFIG_ARM_VIRT_EXT + bl __hyp_stub_install_secondary +#endif + safe_svcmode_maskall r9 + #ifndef CONFIG_CPU_CP15 ldr r9, =CONFIG_PROCESSOR_ID #else @@ -177,7 +185,7 @@ M_CLASS(streq r3, [r12, #PMSAv8_MAIR1]) bic r0, r0, #CR_I #endif mcr p15, 0, r0, c1, c0, 0 @ write control reg - isb + instr_sync #elif defined (CONFIG_CPU_V7M) #ifdef CONFIG_ARM_MPU ldreq r3, [r12, MPU_CTRL] diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 629e25152c0d..1d5fbf1d1c67 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -456,14 +456,13 @@ static int get_hbp_len(u8 hbp_len) /* * Check whether bp virtual address is in kernel space. */ -int arch_check_bp_in_kernelspace(struct perf_event *bp) +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { unsigned int len; unsigned long va; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - va = info->address; - len = get_hbp_len(info->ctrl.len); + va = hw->address; + len = get_hbp_len(hw->ctrl.len); return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); } @@ -518,42 +517,42 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, /* * Construct an arch_hw_breakpoint from a perf_event. */ -static int arch_build_bp_info(struct perf_event *bp) +static int arch_build_bp_info(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - /* Type */ - switch (bp->attr.bp_type) { + switch (attr->bp_type) { case HW_BREAKPOINT_X: - info->ctrl.type = ARM_BREAKPOINT_EXECUTE; + hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; break; case HW_BREAKPOINT_R: - info->ctrl.type = ARM_BREAKPOINT_LOAD; + hw->ctrl.type = ARM_BREAKPOINT_LOAD; break; case HW_BREAKPOINT_W: - info->ctrl.type = ARM_BREAKPOINT_STORE; + hw->ctrl.type = ARM_BREAKPOINT_STORE; break; case HW_BREAKPOINT_RW: - info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; + hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; break; default: return -EINVAL; } /* Len */ - switch (bp->attr.bp_len) { + switch (attr->bp_len) { case HW_BREAKPOINT_LEN_1: - info->ctrl.len = ARM_BREAKPOINT_LEN_1; + hw->ctrl.len = ARM_BREAKPOINT_LEN_1; break; case HW_BREAKPOINT_LEN_2: - info->ctrl.len = ARM_BREAKPOINT_LEN_2; + hw->ctrl.len = ARM_BREAKPOINT_LEN_2; break; case HW_BREAKPOINT_LEN_4: - info->ctrl.len = ARM_BREAKPOINT_LEN_4; + hw->ctrl.len = ARM_BREAKPOINT_LEN_4; break; case HW_BREAKPOINT_LEN_8: - info->ctrl.len = ARM_BREAKPOINT_LEN_8; - if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE) + hw->ctrl.len = ARM_BREAKPOINT_LEN_8; + if ((hw->ctrl.type != ARM_BREAKPOINT_EXECUTE) && max_watchpoint_len >= 8) break; default: @@ -566,24 +565,24 @@ static int arch_build_bp_info(struct perf_event *bp) * by the hardware and must be aligned to the appropriate number of * bytes. */ - if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE && - info->ctrl.len != ARM_BREAKPOINT_LEN_2 && - info->ctrl.len != ARM_BREAKPOINT_LEN_4) + if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE && + hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && + hw->ctrl.len != ARM_BREAKPOINT_LEN_4) return -EINVAL; /* Address */ - info->address = bp->attr.bp_addr; + hw->address = attr->bp_addr; /* Privilege */ - info->ctrl.privilege = ARM_BREAKPOINT_USER; - if (arch_check_bp_in_kernelspace(bp)) - info->ctrl.privilege |= ARM_BREAKPOINT_PRIV; + hw->ctrl.privilege = ARM_BREAKPOINT_USER; + if (arch_check_bp_in_kernelspace(hw)) + hw->ctrl.privilege |= ARM_BREAKPOINT_PRIV; /* Enabled? */ - info->ctrl.enabled = !bp->attr.disabled; + hw->ctrl.enabled = !attr->disabled; /* Mismatch */ - info->ctrl.mismatch = 0; + hw->ctrl.mismatch = 0; return 0; } @@ -591,9 +590,10 @@ static int arch_build_bp_info(struct perf_event *bp) /* * Validate the arch-specific HW Breakpoint register settings. */ -int arch_validate_hwbkpt_settings(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); int ret = 0; u32 offset, alignment_mask = 0x3; @@ -602,14 +602,14 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) return -ENODEV; /* Build the arch_hw_breakpoint. */ - ret = arch_build_bp_info(bp); + ret = arch_build_bp_info(bp, attr, hw); if (ret) goto out; /* Check address alignment. */ - if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) alignment_mask = 0x7; - offset = info->address & alignment_mask; + offset = hw->address & alignment_mask; switch (offset) { case 0: /* Aligned */ @@ -617,19 +617,19 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) case 1: case 2: /* Allow halfword watchpoints and breakpoints. */ - if (info->ctrl.len == ARM_BREAKPOINT_LEN_2) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) break; case 3: /* Allow single byte watchpoint. */ - if (info->ctrl.len == ARM_BREAKPOINT_LEN_1) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) break; default: ret = -EINVAL; goto out; } - info->address &= ~alignment_mask; - info->ctrl.len <<= offset; + hw->address &= ~alignment_mask; + hw->ctrl.len <<= offset; if (is_default_overflow_handler(bp)) { /* @@ -640,7 +640,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) return -EINVAL; /* We don't allow mismatch breakpoints in kernel space. */ - if (arch_check_bp_in_kernelspace(bp)) + if (arch_check_bp_in_kernelspace(hw)) return -EPERM; /* @@ -655,8 +655,8 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) * reports them. */ if (!debug_exception_updates_fsr() && - (info->ctrl.type == ARM_BREAKPOINT_LOAD || - info->ctrl.type == ARM_BREAKPOINT_STORE)) + (hw->ctrl.type == ARM_BREAKPOINT_LOAD || + hw->ctrl.type == ARM_BREAKPOINT_STORE)) return -EINVAL; } diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index ece04a457486..9908dacf9229 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -102,16 +102,6 @@ void __init init_IRQ(void) uniphier_cache_init(); } -#ifdef CONFIG_MULTI_IRQ_HANDLER -void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) -{ - if (handle_arch_irq) - return; - - handle_arch_irq = handle_irq; -} -#endif - #ifdef CONFIG_SPARSE_IRQ int __init arch_probe_nr_irqs(void) { diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 225d1c58d2de..d9c299133111 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -338,6 +338,7 @@ static struct vm_area_struct gate_vma = { static int __init gate_vma_init(void) { + vma_init(&gate_vma, NULL); gate_vma.vm_page_prot = PAGE_READONLY_EXEC; return 0; } diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 35ca494c028c..4c249cb261f3 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1145,7 +1145,7 @@ void __init setup_arch(char **cmdline_p) reserve_crashkernel(); -#ifdef CONFIG_MULTI_IRQ_HANDLER +#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER handle_arch_irq = mdesc->handle_irq; #endif diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index dec130e7078c..b8f766cf3a90 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -150,22 +150,18 @@ static int preserve_vfp_context(struct vfp_sigframe __user *frame) static int restore_vfp_context(char __user **auxp) { - struct vfp_sigframe __user *frame = - (struct vfp_sigframe __user *)*auxp; - unsigned long magic; - unsigned long size; - int err = 0; - - __get_user_error(magic, &frame->magic, err); - __get_user_error(size, &frame->size, err); + struct vfp_sigframe frame; + int err; + err = __copy_from_user(&frame, *auxp, sizeof(frame)); if (err) - return -EFAULT; - if (magic != VFP_MAGIC || size != VFP_STORAGE_SIZE) + return err; + + if (frame.magic != VFP_MAGIC || frame.size != VFP_STORAGE_SIZE) return -EINVAL; - *auxp += size; - return vfp_restore_user_hwstate(&frame->ufp, &frame->ufp_exc); + *auxp += sizeof(frame); + return vfp_restore_user_hwstate(&frame.ufp, &frame.ufp_exc); } #endif @@ -176,6 +172,7 @@ static int restore_vfp_context(char __user **auxp) static int restore_sigframe(struct pt_regs *regs, struct sigframe __user *sf) { + struct sigcontext context; char __user *aux; sigset_t set; int err; @@ -184,23 +181,26 @@ static int restore_sigframe(struct pt_regs *regs, struct sigframe __user *sf) if (err == 0) set_current_blocked(&set); - __get_user_error(regs->ARM_r0, &sf->uc.uc_mcontext.arm_r0, err); - __get_user_error(regs->ARM_r1, &sf->uc.uc_mcontext.arm_r1, err); - __get_user_error(regs->ARM_r2, &sf->uc.uc_mcontext.arm_r2, err); - __get_user_error(regs->ARM_r3, &sf->uc.uc_mcontext.arm_r3, err); - __get_user_error(regs->ARM_r4, &sf->uc.uc_mcontext.arm_r4, err); - __get_user_error(regs->ARM_r5, &sf->uc.uc_mcontext.arm_r5, err); - __get_user_error(regs->ARM_r6, &sf->uc.uc_mcontext.arm_r6, err); - __get_user_error(regs->ARM_r7, &sf->uc.uc_mcontext.arm_r7, err); - __get_user_error(regs->ARM_r8, &sf->uc.uc_mcontext.arm_r8, err); - __get_user_error(regs->ARM_r9, &sf->uc.uc_mcontext.arm_r9, err); - __get_user_error(regs->ARM_r10, &sf->uc.uc_mcontext.arm_r10, err); - __get_user_error(regs->ARM_fp, &sf->uc.uc_mcontext.arm_fp, err); - __get_user_error(regs->ARM_ip, &sf->uc.uc_mcontext.arm_ip, err); - __get_user_error(regs->ARM_sp, &sf->uc.uc_mcontext.arm_sp, err); - __get_user_error(regs->ARM_lr, &sf->uc.uc_mcontext.arm_lr, err); - __get_user_error(regs->ARM_pc, &sf->uc.uc_mcontext.arm_pc, err); - __get_user_error(regs->ARM_cpsr, &sf->uc.uc_mcontext.arm_cpsr, err); + err |= __copy_from_user(&context, &sf->uc.uc_mcontext, sizeof(context)); + if (err == 0) { + regs->ARM_r0 = context.arm_r0; + regs->ARM_r1 = context.arm_r1; + regs->ARM_r2 = context.arm_r2; + regs->ARM_r3 = context.arm_r3; + regs->ARM_r4 = context.arm_r4; + regs->ARM_r5 = context.arm_r5; + regs->ARM_r6 = context.arm_r6; + regs->ARM_r7 = context.arm_r7; + regs->ARM_r8 = context.arm_r8; + regs->ARM_r9 = context.arm_r9; + regs->ARM_r10 = context.arm_r10; + regs->ARM_fp = context.arm_fp; + regs->ARM_ip = context.arm_ip; + regs->ARM_sp = context.arm_sp; + regs->ARM_lr = context.arm_lr; + regs->ARM_pc = context.arm_pc; + regs->ARM_cpsr = context.arm_cpsr; + } err |= !valid_user_regs(regs); diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 1df21a61e379..f0dd4b6ebb63 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -329,9 +329,11 @@ asmlinkage long sys_oabi_semtimedop(int semid, return -ENOMEM; err = 0; for (i = 0; i < nsops; i++) { - __get_user_error(sops[i].sem_num, &tsops->sem_num, err); - __get_user_error(sops[i].sem_op, &tsops->sem_op, err); - __get_user_error(sops[i].sem_flg, &tsops->sem_flg, err); + struct oabi_sembuf osb; + err |= __copy_from_user(&osb, tsops, sizeof(osb)); + sops[i].sem_num = osb.sem_num; + sops[i].sem_op = osb.sem_op; + sops[i].sem_flg = osb.sem_flg; tsops++; } if (timeout) { diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c index cf2701cb0de8..078b259ead4e 100644 --- a/arch/arm/kernel/time.c +++ b/arch/arm/kernel/time.c @@ -83,29 +83,18 @@ static void dummy_clock_access(struct timespec64 *ts) } static clock_access_fn __read_persistent_clock = dummy_clock_access; -static clock_access_fn __read_boot_clock = dummy_clock_access; void read_persistent_clock64(struct timespec64 *ts) { __read_persistent_clock(ts); } -void read_boot_clock64(struct timespec64 *ts) -{ - __read_boot_clock(ts); -} - -int __init register_persistent_clock(clock_access_fn read_boot, - clock_access_fn read_persistent) +int __init register_persistent_clock(clock_access_fn read_persistent) { /* Only allow the clockaccess functions to be registered once */ - if (__read_persistent_clock == dummy_clock_access && - __read_boot_clock == dummy_clock_access) { - if (read_boot) - __read_boot_clock = read_boot; + if (__read_persistent_clock == dummy_clock_access) { if (read_persistent) __read_persistent_clock = read_persistent; - return 0; } diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 7a4b06049001..a826df3d3814 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -90,6 +90,15 @@ .text ENTRY(arm_copy_from_user) +#ifdef CONFIG_CPU_SPECTRE + get_thread_info r3 + ldr r3, [r3, #TI_ADDR_LIMIT] + adds ip, r1, r2 @ ip=addr+size + sub r3, r3, #1 @ addr_limit - 1 + cmpcc ip, r3 @ if (addr+size > addr_limit - 1) + movcs r1, #0 @ addr = NULL + csdb +#endif #include "copy_template.S" diff --git a/arch/arm/mach-omap2/omap-smp.c b/arch/arm/mach-omap2/omap-smp.c index 69df3620eca5..1c73694c871a 100644 --- a/arch/arm/mach-omap2/omap-smp.c +++ b/arch/arm/mach-omap2/omap-smp.c @@ -109,6 +109,45 @@ void omap5_erratum_workaround_801819(void) static inline void omap5_erratum_workaround_801819(void) { } #endif +#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR +/* + * Configure ACR and enable ACTLR[0] (Enable invalidates of BTB with + * ICIALLU) to activate the workaround for secondary Core. + * NOTE: it is assumed that the primary core's configuration is done + * by the boot loader (kernel will detect a misconfiguration and complain + * if this is not done). + * + * In General Purpose(GP) devices, ACR bit settings can only be done + * by ROM code in "secure world" using the smc call and there is no + * option to update the "firmware" on such devices. This also works for + * High security(HS) devices, as a backup option in case the + * "update" is not done in the "security firmware". + */ +static void omap5_secondary_harden_predictor(void) +{ + u32 acr, acr_mask; + + asm volatile ("mrc p15, 0, %0, c1, c0, 1" : "=r" (acr)); + + /* + * ACTLR[0] (Enable invalidates of BTB with ICIALLU) + */ + acr_mask = BIT(0); + + /* Do we already have it done.. if yes, skip expensive smc */ + if ((acr & acr_mask) == acr_mask) + return; + + acr |= acr_mask; + omap_smc1(OMAP5_DRA7_MON_SET_ACR_INDEX, acr); + + pr_debug("%s: ARM ACR setup for CVE_2017_5715 applied on CPU%d\n", + __func__, smp_processor_id()); +} +#else +static inline void omap5_secondary_harden_predictor(void) { } +#endif + static void omap4_secondary_init(unsigned int cpu) { /* @@ -131,6 +170,8 @@ static void omap4_secondary_init(unsigned int cpu) set_cntfreq(); /* Configure ACR to disable streaming WA for 801819 */ omap5_erratum_workaround_801819(); + /* Enable ACR to allow for ICUALLU workaround */ + omap5_secondary_harden_predictor(); } /* diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c index 9c10248fadcc..4e8c2116808e 100644 --- a/arch/arm/mach-pxa/irq.c +++ b/arch/arm/mach-pxa/irq.c @@ -185,7 +185,7 @@ static int pxa_irq_suspend(void) { int i; - for (i = 0; i < pxa_internal_irq_nr / 32; i++) { + for (i = 0; i < DIV_ROUND_UP(pxa_internal_irq_nr, 32); i++) { void __iomem *base = irq_base(i); saved_icmr[i] = __raw_readl(base + ICMR); @@ -204,7 +204,7 @@ static void pxa_irq_resume(void) { int i; - for (i = 0; i < pxa_internal_irq_nr / 32; i++) { + for (i = 0; i < DIV_ROUND_UP(pxa_internal_irq_nr, 32); i++) { void __iomem *base = irq_base(i); __raw_writel(saved_icmr[i], base + ICMR); diff --git a/arch/arm/mach-rpc/ecard.c b/arch/arm/mach-rpc/ecard.c index 39aef4876ed4..04b2f22c2739 100644 --- a/arch/arm/mach-rpc/ecard.c +++ b/arch/arm/mach-rpc/ecard.c @@ -212,7 +212,7 @@ static DEFINE_MUTEX(ecard_mutex); */ static void ecard_init_pgtables(struct mm_struct *mm) { - struct vm_area_struct vma; + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, VM_EXEC); /* We want to set up the page tables for the following mapping: * Virtual Physical @@ -237,9 +237,6 @@ static void ecard_init_pgtables(struct mm_struct *mm) memcpy(dst_pgd, src_pgd, sizeof(pgd_t) * (EASI_SIZE / PGDIR_SIZE)); - vma.vm_flags = VM_EXEC; - vma.vm_mm = mm; - flush_tlb_range(&vma, IO_START, IO_START + IO_SIZE); flush_tlb_range(&vma, EASI_START, EASI_START + EASI_SIZE); } diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 96a7b6cf459b..b169e580bf82 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -702,7 +702,6 @@ config ARM_THUMBEE config ARM_VIRT_EXT bool - depends on MMU default y if CPU_V7 help Enable the kernel to make use of the ARM Virtualization diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index c186474422f3..0cc8e04295a4 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -736,20 +736,29 @@ static int __mark_rodata_ro(void *unused) return 0; } +static int kernel_set_to_readonly __read_mostly; + void mark_rodata_ro(void) { + kernel_set_to_readonly = 1; stop_machine(__mark_rodata_ro, NULL, NULL); debug_checkwx(); } void set_kernel_text_rw(void) { + if (!kernel_set_to_readonly) + return; + set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, current->active_mm); } void set_kernel_text_ro(void) { + if (!kernel_set_to_readonly) + return; + set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, current->active_mm); } diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 5dd6c58d653b..7d67c70bbded 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -53,7 +53,8 @@ static inline bool security_extensions_enabled(void) { /* Check CPUID Identification Scheme before ID_PFR1 read */ if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) - return !!cpuid_feature_extract(CPUID_EXT_PFR1, 4); + return cpuid_feature_extract(CPUID_EXT_PFR1, 4) || + cpuid_feature_extract(CPUID_EXT_PFR1, 20); return 0; } diff --git a/arch/arm/mm/tcm.h b/arch/arm/mm/tcm.h index 8015ad434a40..24101925fe64 100644 --- a/arch/arm/mm/tcm.h +++ b/arch/arm/mm/tcm.h @@ -11,7 +11,7 @@ void __init tcm_init(void); #else /* No TCM support, just blank inlines to be optimized out */ -inline void tcm_init(void) +static inline void tcm_init(void) { } #endif diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 6e8b71613039..f6a62ae44a65 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1844,7 +1844,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) /* there are 2 passes here */ bpf_jit_dump(prog->len, image_size, 2, ctx.target); - set_memory_ro((unsigned long)header, header->pages); + bpf_jit_binary_lock_ro(header); prog->bpf_func = (void *)ctx.target; prog->jited = 1; prog->jited_len = image_size; diff --git a/arch/arm/plat-omap/counter_32k.c b/arch/arm/plat-omap/counter_32k.c index 2438b96004c1..fcc5bfec8bd1 100644 --- a/arch/arm/plat-omap/counter_32k.c +++ b/arch/arm/plat-omap/counter_32k.c @@ -110,7 +110,7 @@ int __init omap_init_clocksource_32k(void __iomem *vbase) } sched_clock_register(omap_32k_read_sched_clock, 32, 32768); - register_persistent_clock(NULL, omap_read_persistent_clock64); + register_persistent_clock(omap_read_persistent_clock64); pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); return 0; diff --git a/arch/arm/probes/kprobes/core.c b/arch/arm/probes/kprobes/core.c index e90cc8a08186..f8bd523d64d1 100644 --- a/arch/arm/probes/kprobes/core.c +++ b/arch/arm/probes/kprobes/core.c @@ -47,9 +47,6 @@ (unsigned long)(addr) + \ (size)) -/* Used as a marker in ARM_pc to note when we're in a jprobe. */ -#define JPROBE_MAGIC_ADDR 0xffffffff - DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); @@ -289,8 +286,8 @@ void __kprobes kprobe_handler(struct pt_regs *regs) break; case KPROBE_REENTER: /* A nested probe was hit in FIQ, it is a BUG */ - pr_warn("Unrecoverable kprobe detected at %p.\n", - p->addr); + pr_warn("Unrecoverable kprobe detected.\n"); + dump_kprobe(p); /* fall through */ default: /* impossible cases */ @@ -303,10 +300,10 @@ void __kprobes kprobe_handler(struct pt_regs *regs) /* * If we have no pre-handler or it returned 0, we - * continue with normal processing. If we have a - * pre-handler and it returned non-zero, it prepped - * for calling the break_handler below on re-entry, - * so get out doing nothing more here. + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. */ if (!p->pre_handler || !p->pre_handler(p, regs)) { kcb->kprobe_status = KPROBE_HIT_SS; @@ -315,20 +312,9 @@ void __kprobes kprobe_handler(struct pt_regs *regs) kcb->kprobe_status = KPROBE_HIT_SSDONE; p->post_handler(p, regs, 0); } - reset_current_kprobe(); - } - } - } else if (cur) { - /* We probably hit a jprobe. Call its break handler. */ - if (cur->break_handler && cur->break_handler(cur, regs)) { - kcb->kprobe_status = KPROBE_HIT_SS; - singlestep(cur, regs, kcb); - if (cur->post_handler) { - kcb->kprobe_status = KPROBE_HIT_SSDONE; - cur->post_handler(cur, regs, 0); } + reset_current_kprobe(); } - reset_current_kprobe(); } else { /* * The probe was removed and a race is in progress. @@ -521,117 +507,6 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, regs->ARM_lr = (unsigned long)&kretprobe_trampoline; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - long sp_addr = regs->ARM_sp; - long cpsr; - - kcb->jprobe_saved_regs = *regs; - memcpy(kcb->jprobes_stack, (void *)sp_addr, MIN_STACK_SIZE(sp_addr)); - regs->ARM_pc = (long)jp->entry; - - cpsr = regs->ARM_cpsr | PSR_I_BIT; -#ifdef CONFIG_THUMB2_KERNEL - /* Set correct Thumb state in cpsr */ - if (regs->ARM_pc & 1) - cpsr |= PSR_T_BIT; - else - cpsr &= ~PSR_T_BIT; -#endif - regs->ARM_cpsr = cpsr; - - preempt_disable(); - return 1; -} - -void __kprobes jprobe_return(void) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - __asm__ __volatile__ ( - /* - * Setup an empty pt_regs. Fill SP and PC fields as - * they're needed by longjmp_break_handler. - * - * We allocate some slack between the original SP and start of - * our fabricated regs. To be precise we want to have worst case - * covered which is STMFD with all 16 regs so we allocate 2 * - * sizeof(struct_pt_regs)). - * - * This is to prevent any simulated instruction from writing - * over the regs when they are accessing the stack. - */ -#ifdef CONFIG_THUMB2_KERNEL - "sub r0, %0, %1 \n\t" - "mov sp, r0 \n\t" -#else - "sub sp, %0, %1 \n\t" -#endif - "ldr r0, ="__stringify(JPROBE_MAGIC_ADDR)"\n\t" - "str %0, [sp, %2] \n\t" - "str r0, [sp, %3] \n\t" - "mov r0, sp \n\t" - "bl kprobe_handler \n\t" - - /* - * Return to the context saved by setjmp_pre_handler - * and restored by longjmp_break_handler. - */ -#ifdef CONFIG_THUMB2_KERNEL - "ldr lr, [sp, %2] \n\t" /* lr = saved sp */ - "ldrd r0, r1, [sp, %5] \n\t" /* r0,r1 = saved lr,pc */ - "ldr r2, [sp, %4] \n\t" /* r2 = saved psr */ - "stmdb lr!, {r0, r1, r2} \n\t" /* push saved lr and */ - /* rfe context */ - "ldmia sp, {r0 - r12} \n\t" - "mov sp, lr \n\t" - "ldr lr, [sp], #4 \n\t" - "rfeia sp! \n\t" -#else - "ldr r0, [sp, %4] \n\t" - "msr cpsr_cxsf, r0 \n\t" - "ldmia sp, {r0 - pc} \n\t" -#endif - : - : "r" (kcb->jprobe_saved_regs.ARM_sp), - "I" (sizeof(struct pt_regs) * 2), - "J" (offsetof(struct pt_regs, ARM_sp)), - "J" (offsetof(struct pt_regs, ARM_pc)), - "J" (offsetof(struct pt_regs, ARM_cpsr)), - "J" (offsetof(struct pt_regs, ARM_lr)) - : "memory", "cc"); -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - long stack_addr = kcb->jprobe_saved_regs.ARM_sp; - long orig_sp = regs->ARM_sp; - struct jprobe *jp = container_of(p, struct jprobe, kp); - - if (regs->ARM_pc == JPROBE_MAGIC_ADDR) { - if (orig_sp != stack_addr) { - struct pt_regs *saved_regs = - (struct pt_regs *)kcb->jprobe_saved_regs.ARM_sp; - printk("current sp %lx does not match saved sp %lx\n", - orig_sp, stack_addr); - printk("Saved registers for jprobe %p\n", jp); - show_regs(saved_regs); - printk("Current registers\n"); - show_regs(regs); - BUG(); - } - *regs = kcb->jprobe_saved_regs; - memcpy((void *)stack_addr, kcb->jprobes_stack, - MIN_STACK_SIZE(stack_addr)); - preempt_enable_no_resched(); - return 1; - } - return 0; -} - int __kprobes arch_trampoline_kprobe(struct kprobe *p) { return 0; diff --git a/arch/arm/probes/kprobes/test-core.c b/arch/arm/probes/kprobes/test-core.c index 14db14152909..cc237fa9b90f 100644 --- a/arch/arm/probes/kprobes/test-core.c +++ b/arch/arm/probes/kprobes/test-core.c @@ -1461,7 +1461,6 @@ fail: print_registers(&result_regs); if (mem) { - pr_err("current_stack=%p\n", current_stack); pr_err("expected_memory:\n"); print_memory(expected_memory, mem_size); pr_err("result_memory:\n"); diff --git a/arch/arm/vfp/Makefile b/arch/arm/vfp/Makefile index a81404c09d5d..94516c40ebd3 100644 --- a/arch/arm/vfp/Makefile +++ b/arch/arm/vfp/Makefile @@ -8,8 +8,5 @@ # asflags-y := -DDEBUG KBUILD_AFLAGS :=$(KBUILD_AFLAGS:-msoft-float=-Wa,-mfpu=softvfp+vfp -mfloat-abi=soft) -LDFLAGS +=--no-warn-mismatch -obj-y += vfp.o - -vfp-$(CONFIG_VFP) += vfpmodule.o entry.o vfphw.o vfpsingle.o vfpdouble.o +obj-y += vfpmodule.o entry.o vfphw.o vfpsingle.o vfpdouble.o diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 35d0f823e823..dc7e6b50ef67 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -596,13 +596,11 @@ int vfp_preserve_user_clear_hwstate(struct user_vfp __user *ufp, } /* Sanitise and restore the current VFP state from the provided structures. */ -int vfp_restore_user_hwstate(struct user_vfp __user *ufp, - struct user_vfp_exc __user *ufp_exc) +int vfp_restore_user_hwstate(struct user_vfp *ufp, struct user_vfp_exc *ufp_exc) { struct thread_info *thread = current_thread_info(); struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; unsigned long fpexc; - int err = 0; /* Disable VFP to avoid corrupting the new thread state. */ vfp_flush_hwstate(thread); @@ -611,17 +609,16 @@ int vfp_restore_user_hwstate(struct user_vfp __user *ufp, * Copy the floating point registers. There can be unused * registers see asm/hwcap.h for details. */ - err |= __copy_from_user(&hwstate->fpregs, &ufp->fpregs, - sizeof(hwstate->fpregs)); + memcpy(&hwstate->fpregs, &ufp->fpregs, sizeof(hwstate->fpregs)); /* * Copy the status and control register. */ - __get_user_error(hwstate->fpscr, &ufp->fpscr, err); + hwstate->fpscr = ufp->fpscr; /* * Sanitise and restore the exception registers. */ - __get_user_error(fpexc, &ufp_exc->fpexc, err); + fpexc = ufp_exc->fpexc; /* Ensure the VFP is enabled. */ fpexc |= FPEXC_EN; @@ -630,10 +627,10 @@ int vfp_restore_user_hwstate(struct user_vfp __user *ufp, fpexc &= ~(FPEXC_EX | FPEXC_FP2V); hwstate->fpexc = fpexc; - __get_user_error(hwstate->fpinst, &ufp_exc->fpinst, err); - __get_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err); + hwstate->fpinst = ufp_exc->fpinst; + hwstate->fpinst2 = ufp_exc->fpinst2; - return err ? -EFAULT : 0; + return 0; } /* diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 42c090cf0292..3d1011957823 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -74,6 +74,7 @@ config ARM64 select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL @@ -264,9 +265,6 @@ config ARCH_SUPPORTS_UPROBES config ARCH_PROC_KCORE_TEXT def_bool y -config MULTI_IRQ_HANDLER - def_bool y - source "init/Kconfig" source "kernel/Kconfig.freezer" diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 45272266dafb..e7101b19d590 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -10,7 +10,7 @@ # # Copyright (C) 1995-2001 by Russell King -LDFLAGS_vmlinux :=-p --no-undefined -X +LDFLAGS_vmlinux :=--no-undefined -X CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) GZFLAGS :=-9 @@ -60,15 +60,15 @@ ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian CHECKFLAGS += -D__AARCH64EB__ AS += -EB -LD += -EB -LDFLAGS += -maarch64linuxb +# We must use the linux target here, since distributions don't tend to package +# the ELF linker scripts with binutils, and this results in a build failure. +LDFLAGS += -EB -maarch64linuxb UTS_MACHINE := aarch64_be else KBUILD_CPPFLAGS += -mlittle-endian CHECKFLAGS += -D__AARCH64EL__ AS += -EL -LD += -EL -LDFLAGS += -maarch64linux +LDFLAGS += -EL -maarch64linux # See comment above UTS_MACHINE := aarch64 endif diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 88f5aef7934c..e3a375c4cb83 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -19,33 +19,24 @@ * u32 *macp, u8 const rk[], u32 rounds); */ ENTRY(ce_aes_ccm_auth_data) - frame_push 7 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x5 - - ldr w25, [x22] /* leftover from prev round? */ + ldr w8, [x3] /* leftover from prev round? */ ld1 {v0.16b}, [x0] /* load mac */ - cbz w25, 1f - sub w25, w25, #16 + cbz w8, 1f + sub w8, w8, #16 eor v1.16b, v1.16b, v1.16b -0: ldrb w7, [x20], #1 /* get 1 byte of input */ - subs w21, w21, #1 - add w25, w25, #1 +0: ldrb w7, [x1], #1 /* get 1 byte of input */ + subs w2, w2, #1 + add w8, w8, #1 ins v1.b[0], w7 ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */ beq 8f /* out of input? */ - cbnz w25, 0b + cbnz w8, 0b eor v0.16b, v0.16b, v1.16b -1: ld1 {v3.4s}, [x23] /* load first round key */ - prfm pldl1strm, [x20] - cmp w24, #12 /* which key size? */ - add x6, x23, #16 - sub w7, w24, #2 /* modified # of rounds */ +1: ld1 {v3.4s}, [x4] /* load first round key */ + prfm pldl1strm, [x1] + cmp w5, #12 /* which key size? */ + add x6, x4, #16 + sub w7, w5, #2 /* modified # of rounds */ bmi 2f bne 5f mov v5.16b, v3.16b @@ -64,43 +55,33 @@ ENTRY(ce_aes_ccm_auth_data) ld1 {v5.4s}, [x6], #16 /* load next round key */ bpl 3b aese v0.16b, v4.16b - subs w21, w21, #16 /* last data? */ + subs w2, w2, #16 /* last data? */ eor v0.16b, v0.16b, v5.16b /* final round */ bmi 6f - ld1 {v1.16b}, [x20], #16 /* load next input block */ + ld1 {v1.16b}, [x1], #16 /* load next input block */ eor v0.16b, v0.16b, v1.16b /* xor with mac */ - beq 6f - - if_will_cond_yield_neon - st1 {v0.16b}, [x19] /* store mac */ - do_cond_yield_neon - ld1 {v0.16b}, [x19] /* reload mac */ - endif_yield_neon - - b 1b -6: st1 {v0.16b}, [x19] /* store mac */ + bne 1b +6: st1 {v0.16b}, [x0] /* store mac */ beq 10f - adds w21, w21, #16 + adds w2, w2, #16 beq 10f - mov w25, w21 -7: ldrb w7, [x20], #1 + mov w8, w2 +7: ldrb w7, [x1], #1 umov w6, v0.b[0] eor w6, w6, w7 - strb w6, [x19], #1 - subs w21, w21, #1 + strb w6, [x0], #1 + subs w2, w2, #1 beq 10f ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ b 7b -8: mov w7, w25 - add w25, w25, #16 +8: mov w7, w8 + add w8, w8, #16 9: ext v1.16b, v1.16b, v1.16b, #1 adds w7, w7, #1 bne 9b eor v0.16b, v0.16b, v1.16b - st1 {v0.16b}, [x19] -10: str w25, [x22] - - frame_pop + st1 {v0.16b}, [x0] +10: str w8, [x3] ret ENDPROC(ce_aes_ccm_auth_data) @@ -145,29 +126,19 @@ ENTRY(ce_aes_ccm_final) ENDPROC(ce_aes_ccm_final) .macro aes_ccm_do_crypt,enc - frame_push 8 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x5 - mov x25, x6 - - ldr x26, [x25, #8] /* load lower ctr */ - ld1 {v0.16b}, [x24] /* load mac */ -CPU_LE( rev x26, x26 ) /* keep swabbed ctr in reg */ + ldr x8, [x6, #8] /* load lower ctr */ + ld1 {v0.16b}, [x5] /* load mac */ +CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ 0: /* outer loop */ - ld1 {v1.8b}, [x25] /* load upper ctr */ - prfm pldl1strm, [x20] - add x26, x26, #1 - rev x9, x26 - cmp w23, #12 /* which key size? */ - sub w7, w23, #2 /* get modified # of rounds */ + ld1 {v1.8b}, [x6] /* load upper ctr */ + prfm pldl1strm, [x1] + add x8, x8, #1 + rev x9, x8 + cmp w4, #12 /* which key size? */ + sub w7, w4, #2 /* get modified # of rounds */ ins v1.d[1], x9 /* no carry in lower ctr */ - ld1 {v3.4s}, [x22] /* load first round key */ - add x10, x22, #16 + ld1 {v3.4s}, [x3] /* load first round key */ + add x10, x3, #16 bmi 1f bne 4f mov v5.16b, v3.16b @@ -194,9 +165,9 @@ CPU_LE( rev x26, x26 ) /* keep swabbed ctr in reg */ bpl 2b aese v0.16b, v4.16b aese v1.16b, v4.16b - subs w21, w21, #16 - bmi 7f /* partial block? */ - ld1 {v2.16b}, [x20], #16 /* load next input block */ + subs w2, w2, #16 + bmi 6f /* partial block? */ + ld1 {v2.16b}, [x1], #16 /* load next input block */ .if \enc == 1 eor v2.16b, v2.16b, v5.16b /* final round enc+mac */ eor v1.16b, v1.16b, v2.16b /* xor with crypted ctr */ @@ -205,29 +176,18 @@ CPU_LE( rev x26, x26 ) /* keep swabbed ctr in reg */ eor v1.16b, v2.16b, v5.16b /* final round enc */ .endif eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ - st1 {v1.16b}, [x19], #16 /* write output block */ - beq 5f - - if_will_cond_yield_neon - st1 {v0.16b}, [x24] /* store mac */ - do_cond_yield_neon - ld1 {v0.16b}, [x24] /* reload mac */ - endif_yield_neon - - b 0b -5: -CPU_LE( rev x26, x26 ) - st1 {v0.16b}, [x24] /* store mac */ - str x26, [x25, #8] /* store lsb end of ctr (BE) */ - -6: frame_pop - ret - -7: eor v0.16b, v0.16b, v5.16b /* final round mac */ + st1 {v1.16b}, [x0], #16 /* write output block */ + bne 0b +CPU_LE( rev x8, x8 ) + st1 {v0.16b}, [x5] /* store mac */ + str x8, [x6, #8] /* store lsb end of ctr (BE) */ +5: ret + +6: eor v0.16b, v0.16b, v5.16b /* final round mac */ eor v1.16b, v1.16b, v5.16b /* final round enc */ - st1 {v0.16b}, [x24] /* store mac */ - add w21, w21, #16 /* process partial tail block */ -8: ldrb w9, [x20], #1 /* get 1 byte of input */ + st1 {v0.16b}, [x5] /* store mac */ + add w2, w2, #16 /* process partial tail block */ +7: ldrb w9, [x1], #1 /* get 1 byte of input */ umov w6, v1.b[0] /* get top crypted ctr byte */ umov w7, v0.b[0] /* get top mac byte */ .if \enc == 1 @@ -237,13 +197,13 @@ CPU_LE( rev x26, x26 ) eor w9, w9, w6 eor w7, w7, w9 .endif - strb w9, [x19], #1 /* store out byte */ - strb w7, [x24], #1 /* store mac byte */ - subs w21, w21, #1 - beq 6b + strb w9, [x0], #1 /* store out byte */ + strb w7, [x5], #1 /* store mac byte */ + subs w2, w2, #1 + beq 5b ext v0.16b, v0.16b, v0.16b, #1 /* shift out mac byte */ ext v1.16b, v1.16b, v1.16b, #1 /* shift out ctr byte */ - b 8b + b 7b .endm /* diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index dcffb9e77589..c723647b37db 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -322,55 +322,41 @@ ENDPROC(pmull_ghash_update_p8) .endm .macro pmull_gcm_do_crypt, enc - frame_push 10 + ld1 {SHASH.2d}, [x4] + ld1 {XL.2d}, [x1] + ldr x8, [x5, #8] // load lower counter - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x5 - mov x25, x6 - mov x26, x7 - .if \enc == 1 - ldr x27, [sp, #96] // first stacked arg - .endif - - ldr x28, [x24, #8] // load lower counter -CPU_LE( rev x28, x28 ) - -0: mov x0, x25 - load_round_keys w26, x0 - ld1 {SHASH.2d}, [x23] - ld1 {XL.2d}, [x20] + load_round_keys w7, x6 movi MASK.16b, #0xe1 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 +CPU_LE( rev x8, x8 ) shl MASK.2d, MASK.2d, #57 eor SHASH2.16b, SHASH2.16b, SHASH.16b .if \enc == 1 - ld1 {KS.16b}, [x27] + ldr x10, [sp] + ld1 {KS.16b}, [x10] .endif -1: ld1 {CTR.8b}, [x24] // load upper counter - ld1 {INP.16b}, [x22], #16 - rev x9, x28 - add x28, x28, #1 - sub w19, w19, #1 +0: ld1 {CTR.8b}, [x5] // load upper counter + ld1 {INP.16b}, [x3], #16 + rev x9, x8 + add x8, x8, #1 + sub w0, w0, #1 ins CTR.d[1], x9 // set lower counter .if \enc == 1 eor INP.16b, INP.16b, KS.16b // encrypt input - st1 {INP.16b}, [x21], #16 + st1 {INP.16b}, [x2], #16 .endif rev64 T1.16b, INP.16b - cmp w26, #12 - b.ge 4f // AES-192/256? + cmp w7, #12 + b.ge 2f // AES-192/256? -2: enc_round CTR, v21 +1: enc_round CTR, v21 ext T2.16b, XL.16b, XL.16b, #8 ext IN1.16b, T1.16b, T1.16b, #8 @@ -425,39 +411,27 @@ CPU_LE( rev x28, x28 ) .if \enc == 0 eor INP.16b, INP.16b, KS.16b - st1 {INP.16b}, [x21], #16 + st1 {INP.16b}, [x2], #16 .endif - cbz w19, 3f + cbnz w0, 0b - if_will_cond_yield_neon - st1 {XL.2d}, [x20] - .if \enc == 1 - st1 {KS.16b}, [x27] - .endif - do_cond_yield_neon - b 0b - endif_yield_neon +CPU_LE( rev x8, x8 ) + st1 {XL.2d}, [x1] + str x8, [x5, #8] // store lower counter - b 1b - -3: st1 {XL.2d}, [x20] .if \enc == 1 - st1 {KS.16b}, [x27] + st1 {KS.16b}, [x10] .endif -CPU_LE( rev x28, x28 ) - str x28, [x24, #8] // store lower counter - - frame_pop ret -4: b.eq 5f // AES-192? +2: b.eq 3f // AES-192? enc_round CTR, v17 enc_round CTR, v18 -5: enc_round CTR, v19 +3: enc_round CTR, v19 enc_round CTR, v20 - b 2b + b 1b .endm /* diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 7cf0b1aa6ea8..8a10f1d7199a 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -488,9 +488,13 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); } - if (walk.nbytes) - pmull_gcm_encrypt_block(iv, iv, NULL, + if (walk.nbytes) { + kernel_neon_begin(); + pmull_gcm_encrypt_block(iv, iv, ctx->aes_key.key_enc, num_rounds(&ctx->aes_key)); + kernel_neon_end(); + } + } else { __aes_arm64_encrypt(ctx->aes_key.key_enc, tag, iv, num_rounds(&ctx->aes_key)); diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index c0235e0ff849..9bca54dda75c 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -40,17 +40,6 @@ #include <asm/cmpxchg.h> -#define ___atomic_add_unless(v, a, u, sfx) \ -({ \ - typeof((v)->counter) c, old; \ - \ - c = atomic##sfx##_read(v); \ - while (c != (u) && \ - (old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c) \ - c = old; \ - c; \ - }) - #define ATOMIC_INIT(i) { (i) } #define atomic_read(v) READ_ONCE((v)->counter) @@ -61,21 +50,11 @@ #define atomic_add_return_release atomic_add_return_release #define atomic_add_return atomic_add_return -#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v)) -#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v)) -#define atomic_inc_return_release(v) atomic_add_return_release(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) - #define atomic_sub_return_relaxed atomic_sub_return_relaxed #define atomic_sub_return_acquire atomic_sub_return_acquire #define atomic_sub_return_release atomic_sub_return_release #define atomic_sub_return atomic_sub_return -#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v)) -#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v)) -#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v)) -#define atomic_dec_return(v) atomic_sub_return(1, (v)) - #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed #define atomic_fetch_add_acquire atomic_fetch_add_acquire #define atomic_fetch_add_release atomic_fetch_add_release @@ -119,13 +98,6 @@ cmpxchg_release(&((v)->counter), (old), (new)) #define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new)) -#define atomic_inc(v) atomic_add(1, (v)) -#define atomic_dec(v) atomic_sub(1, (v)) -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) -#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) -#define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0) -#define __atomic_add_unless(v, a, u) ___atomic_add_unless(v, a, u,) #define atomic_andnot atomic_andnot /* @@ -140,21 +112,11 @@ #define atomic64_add_return_release atomic64_add_return_release #define atomic64_add_return atomic64_add_return -#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v)) -#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v)) -#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v)) -#define atomic64_inc_return(v) atomic64_add_return(1, (v)) - #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed #define atomic64_sub_return_acquire atomic64_sub_return_acquire #define atomic64_sub_return_release atomic64_sub_return_release #define atomic64_sub_return atomic64_sub_return -#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v)) -#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v)) -#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v)) -#define atomic64_dec_return(v) atomic64_sub_return(1, (v)) - #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire #define atomic64_fetch_add_release atomic64_fetch_add_release @@ -195,16 +157,9 @@ #define atomic64_cmpxchg_release atomic_cmpxchg_release #define atomic64_cmpxchg atomic_cmpxchg -#define atomic64_inc(v) atomic64_add(1, (v)) -#define atomic64_dec(v) atomic64_sub(1, (v)) -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) -#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0) -#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) -#define atomic64_add_negative(i, v) (atomic64_add_return((i), (v)) < 0) -#define atomic64_add_unless(v, a, u) (___atomic_add_unless(v, a, u, 64) != u) #define atomic64_andnot atomic64_andnot -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) +#define atomic64_dec_if_positive atomic64_dec_if_positive #endif #endif diff --git a/arch/arm64/include/asm/bitops.h b/arch/arm64/include/asm/bitops.h index 9c19594ce7cb..10d536b1af74 100644 --- a/arch/arm64/include/asm/bitops.h +++ b/arch/arm64/include/asm/bitops.h @@ -17,22 +17,11 @@ #define __ASM_BITOPS_H #include <linux/compiler.h> -#include <asm/barrier.h> #ifndef _LINUX_BITOPS_H #error only <linux/bitops.h> can be included directly #endif -/* - * Little endian assembly atomic bitops. - */ -extern void set_bit(int nr, volatile unsigned long *p); -extern void clear_bit(int nr, volatile unsigned long *p); -extern void change_bit(int nr, volatile unsigned long *p); -extern int test_and_set_bit(int nr, volatile unsigned long *p); -extern int test_and_clear_bit(int nr, volatile unsigned long *p); -extern int test_and_change_bit(int nr, volatile unsigned long *p); - #include <asm-generic/bitops/builtin-__ffs.h> #include <asm-generic/bitops/builtin-ffs.h> #include <asm-generic/bitops/builtin-__fls.h> @@ -44,15 +33,11 @@ extern int test_and_change_bit(int nr, volatile unsigned long *p); #include <asm-generic/bitops/sched.h> #include <asm-generic/bitops/hweight.h> -#include <asm-generic/bitops/lock.h> +#include <asm-generic/bitops/atomic.h> +#include <asm-generic/bitops/lock.h> #include <asm-generic/bitops/non-atomic.h> #include <asm-generic/bitops/le.h> - -/* - * Ext2 is defined to use little-endian byte ordering. - */ -#define ext2_set_bit_atomic(lock, nr, p) test_and_set_bit_le(nr, p) -#define ext2_clear_bit_atomic(lock, nr, p) test_and_clear_bit_le(nr, p) +#include <asm-generic/bitops/ext2-atomic-setbit.h> #endif /* __ASM_BITOPS_H */ diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 192d791f1103..7ed320895d1f 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -87,6 +87,9 @@ static inline unsigned long efi_get_max_initrd_addr(unsigned long dram_base, #define efi_call_runtime(f, ...) sys_table_arg->runtime->f(__VA_ARGS__) #define efi_is_64bit() (true) +#define efi_table_attr(table, attr, instance) \ + ((table##_t *)instance)->attr + #define efi_call_proto(protocol, f, instance, ...) \ ((protocol##_t *)instance)->f(instance, ##__VA_ARGS__) diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index 41770766d964..6a53e59ced95 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -119,13 +119,16 @@ static inline void decode_ctrl_reg(u32 reg, struct task_struct; struct notifier_block; +struct perf_event_attr; struct perf_event; struct pmu; extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, int *gen_len, int *gen_type, int *offset); -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index a0fee6985e6a..b2b0c6405eb0 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -8,8 +8,6 @@ struct pt_regs; -extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); - static inline int nr_legacy_irqs(void) { return 0; diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h index 6deb8d726041..d5a44cf859e9 100644 --- a/arch/arm64/include/asm/kprobes.h +++ b/arch/arm64/include/asm/kprobes.h @@ -48,7 +48,6 @@ struct kprobe_ctlblk { unsigned long saved_irqflag; struct prev_kprobe prev_kprobe; struct kprobe_step_ctx ss_ctx; - struct pt_regs jprobe_saved_regs; }; void arch_remove_kprobe(struct kprobe *); diff --git a/arch/arm64/include/asm/simd.h b/arch/arm64/include/asm/simd.h index fa8b3fe932e6..6495cc51246f 100644 --- a/arch/arm64/include/asm/simd.h +++ b/arch/arm64/include/asm/simd.h @@ -29,20 +29,15 @@ DECLARE_PER_CPU(bool, kernel_neon_busy); static __must_check inline bool may_use_simd(void) { /* - * The raw_cpu_read() is racy if called with preemption enabled. - * This is not a bug: kernel_neon_busy is only set when - * preemption is disabled, so we cannot migrate to another CPU - * while it is set, nor can we migrate to a CPU where it is set. - * So, if we find it clear on some CPU then we're guaranteed to - * find it clear on any CPU we could migrate to. - * - * If we are in between kernel_neon_begin()...kernel_neon_end(), - * the flag will be set, but preemption is also disabled, so we - * can't migrate to another CPU and spuriously see it become - * false. + * kernel_neon_busy is only set while preemption is disabled, + * and is clear whenever preemption is enabled. Since + * this_cpu_read() is atomic w.r.t. preemption, kernel_neon_busy + * cannot change under our feet -- if it's set we cannot be + * migrated, and if it's clear we cannot be migrated to a CPU + * where it is set. */ return !in_irq() && !irqs_disabled() && !in_nmi() && - !raw_cpu_read(kernel_neon_busy); + !this_cpu_read(kernel_neon_busy); } #else /* ! CONFIG_KERNEL_MODE_NEON */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index ffdaea7954bb..0ad1cf233470 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -37,7 +37,7 @@ static inline void __tlb_remove_table(void *_table) static inline void tlb_flush(struct mmu_gather *tlb) { - struct vm_area_struct vma = { .vm_mm = tlb->mm, }; + struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); /* * The ASID allocator will either invalidate the ASID or mark diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index f24892a40d2c..c6d80743f4ed 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1351,9 +1351,9 @@ static void __update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, static void update_cpu_capabilities(u16 scope_mask) { - __update_cpu_capabilities(arm64_features, scope_mask, "detected:"); __update_cpu_capabilities(arm64_errata, scope_mask, "enabling workaround for"); + __update_cpu_capabilities(arm64_features, scope_mask, "detected:"); } static int __enable_cpu_capability(void *arg) @@ -1408,8 +1408,8 @@ __enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps, static void __init enable_cpu_capabilities(u16 scope_mask) { - __enable_cpu_capabilities(arm64_features, scope_mask); __enable_cpu_capabilities(arm64_errata, scope_mask); + __enable_cpu_capabilities(arm64_features, scope_mask); } /* diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index 413dbe530da8..8c9644376326 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -343,14 +343,13 @@ static int get_hbp_len(u8 hbp_len) /* * Check whether bp virtual address is in kernel space. */ -int arch_check_bp_in_kernelspace(struct perf_event *bp) +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { unsigned int len; unsigned long va; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - va = info->address; - len = get_hbp_len(info->ctrl.len); + va = hw->address; + len = get_hbp_len(hw->ctrl.len); return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); } @@ -421,53 +420,53 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, /* * Construct an arch_hw_breakpoint from a perf_event. */ -static int arch_build_bp_info(struct perf_event *bp) +static int arch_build_bp_info(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - /* Type */ - switch (bp->attr.bp_type) { + switch (attr->bp_type) { case HW_BREAKPOINT_X: - info->ctrl.type = ARM_BREAKPOINT_EXECUTE; + hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; break; case HW_BREAKPOINT_R: - info->ctrl.type = ARM_BREAKPOINT_LOAD; + hw->ctrl.type = ARM_BREAKPOINT_LOAD; break; case HW_BREAKPOINT_W: - info->ctrl.type = ARM_BREAKPOINT_STORE; + hw->ctrl.type = ARM_BREAKPOINT_STORE; break; case HW_BREAKPOINT_RW: - info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; + hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; break; default: return -EINVAL; } /* Len */ - switch (bp->attr.bp_len) { + switch (attr->bp_len) { case HW_BREAKPOINT_LEN_1: - info->ctrl.len = ARM_BREAKPOINT_LEN_1; + hw->ctrl.len = ARM_BREAKPOINT_LEN_1; break; case HW_BREAKPOINT_LEN_2: - info->ctrl.len = ARM_BREAKPOINT_LEN_2; + hw->ctrl.len = ARM_BREAKPOINT_LEN_2; break; case HW_BREAKPOINT_LEN_3: - info->ctrl.len = ARM_BREAKPOINT_LEN_3; + hw->ctrl.len = ARM_BREAKPOINT_LEN_3; break; case HW_BREAKPOINT_LEN_4: - info->ctrl.len = ARM_BREAKPOINT_LEN_4; + hw->ctrl.len = ARM_BREAKPOINT_LEN_4; break; case HW_BREAKPOINT_LEN_5: - info->ctrl.len = ARM_BREAKPOINT_LEN_5; + hw->ctrl.len = ARM_BREAKPOINT_LEN_5; break; case HW_BREAKPOINT_LEN_6: - info->ctrl.len = ARM_BREAKPOINT_LEN_6; + hw->ctrl.len = ARM_BREAKPOINT_LEN_6; break; case HW_BREAKPOINT_LEN_7: - info->ctrl.len = ARM_BREAKPOINT_LEN_7; + hw->ctrl.len = ARM_BREAKPOINT_LEN_7; break; case HW_BREAKPOINT_LEN_8: - info->ctrl.len = ARM_BREAKPOINT_LEN_8; + hw->ctrl.len = ARM_BREAKPOINT_LEN_8; break; default: return -EINVAL; @@ -478,37 +477,37 @@ static int arch_build_bp_info(struct perf_event *bp) * AArch32 also requires breakpoints of length 2 for Thumb. * Watchpoints can be of length 1, 2, 4 or 8 bytes. */ - if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { + if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) { if (is_compat_bp(bp)) { - if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 && - info->ctrl.len != ARM_BREAKPOINT_LEN_4) + if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && + hw->ctrl.len != ARM_BREAKPOINT_LEN_4) return -EINVAL; - } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) { + } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) { /* * FIXME: Some tools (I'm looking at you perf) assume * that breakpoints should be sizeof(long). This * is nonsense. For now, we fix up the parameter * but we should probably return -EINVAL instead. */ - info->ctrl.len = ARM_BREAKPOINT_LEN_4; + hw->ctrl.len = ARM_BREAKPOINT_LEN_4; } } /* Address */ - info->address = bp->attr.bp_addr; + hw->address = attr->bp_addr; /* * Privilege * Note that we disallow combined EL0/EL1 breakpoints because * that would complicate the stepping code. */ - if (arch_check_bp_in_kernelspace(bp)) - info->ctrl.privilege = AARCH64_BREAKPOINT_EL1; + if (arch_check_bp_in_kernelspace(hw)) + hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1; else - info->ctrl.privilege = AARCH64_BREAKPOINT_EL0; + hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0; /* Enabled? */ - info->ctrl.enabled = !bp->attr.disabled; + hw->ctrl.enabled = !attr->disabled; return 0; } @@ -516,14 +515,15 @@ static int arch_build_bp_info(struct perf_event *bp) /* * Validate the arch-specific HW Breakpoint register settings. */ -int arch_validate_hwbkpt_settings(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); int ret; u64 alignment_mask, offset; /* Build the arch_hw_breakpoint. */ - ret = arch_build_bp_info(bp); + ret = arch_build_bp_info(bp, attr, hw); if (ret) return ret; @@ -537,42 +537,42 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) * that here. */ if (is_compat_bp(bp)) { - if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) alignment_mask = 0x7; else alignment_mask = 0x3; - offset = info->address & alignment_mask; + offset = hw->address & alignment_mask; switch (offset) { case 0: /* Aligned */ break; case 1: /* Allow single byte watchpoint. */ - if (info->ctrl.len == ARM_BREAKPOINT_LEN_1) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) break; case 2: /* Allow halfword watchpoints and breakpoints. */ - if (info->ctrl.len == ARM_BREAKPOINT_LEN_2) + if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) break; default: return -EINVAL; } } else { - if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) + if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) alignment_mask = 0x3; else alignment_mask = 0x7; - offset = info->address & alignment_mask; + offset = hw->address & alignment_mask; } - info->address &= ~alignment_mask; - info->ctrl.len <<= offset; + hw->address &= ~alignment_mask; + hw->ctrl.len <<= offset; /* * Disallow per-task kernel breakpoints since these would * complicate the stepping code. */ - if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) + if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) return -EINVAL; return 0; diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 60e5fc661f74..780a12f59a8f 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -42,16 +42,6 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } -void (*handle_arch_irq)(struct pt_regs *) = NULL; - -void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) -{ - if (handle_arch_irq) - return; - - handle_arch_irq = handle_irq; -} - #ifdef CONFIG_VMAP_STACK static void init_irq_stacks(void) { diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index d849d9804011..e78c3ef04d95 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -275,7 +275,7 @@ static int __kprobes reenter_kprobe(struct kprobe *p, break; case KPROBE_HIT_SS: case KPROBE_REENTER: - pr_warn("Unrecoverable kprobe detected at %p.\n", p->addr); + pr_warn("Unrecoverable kprobe detected.\n"); dump_kprobe(p); BUG(); break; @@ -395,9 +395,9 @@ static void __kprobes kprobe_handler(struct pt_regs *regs) /* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a - * pre-handler and it returned non-zero, it prepped - * for calling the break_handler below on re-entry, - * so get out doing nothing more here. + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. * * pre_handler can hit a breakpoint and can step thru * before return, keep PSTATE D-flag enabled until @@ -405,16 +405,8 @@ static void __kprobes kprobe_handler(struct pt_regs *regs) */ if (!p->pre_handler || !p->pre_handler(p, regs)) { setup_singlestep(p, regs, kcb, 0); - return; - } - } - } else if ((le32_to_cpu(*(kprobe_opcode_t *) addr) == - BRK64_OPCODE_KPROBES) && cur_kprobe) { - /* We probably hit a jprobe. Call its break handler. */ - if (cur_kprobe->break_handler && - cur_kprobe->break_handler(cur_kprobe, regs)) { - setup_singlestep(cur_kprobe, regs, kcb, 0); - return; + } else + reset_current_kprobe(); } } /* @@ -465,74 +457,6 @@ kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr) return DBG_HOOK_HANDLED; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - kcb->jprobe_saved_regs = *regs; - /* - * Since we can't be sure where in the stack frame "stacked" - * pass-by-value arguments are stored we just don't try to - * duplicate any of the stack. Do not use jprobes on functions that - * use more than 64 bytes (after padding each to an 8 byte boundary) - * of arguments, or pass individual arguments larger than 16 bytes. - */ - - instruction_pointer_set(regs, (unsigned long) jp->entry); - preempt_disable(); - pause_graph_tracing(); - return 1; -} - -void __kprobes jprobe_return(void) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - /* - * Jprobe handler return by entering break exception, - * encoded same as kprobe, but with following conditions - * -a special PC to identify it from the other kprobes. - * -restore stack addr to original saved pt_regs - */ - asm volatile(" mov sp, %0 \n" - "jprobe_return_break: brk %1 \n" - : - : "r" (kcb->jprobe_saved_regs.sp), - "I" (BRK64_ESR_KPROBES) - : "memory"); - - unreachable(); -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - long stack_addr = kcb->jprobe_saved_regs.sp; - long orig_sp = kernel_stack_pointer(regs); - struct jprobe *jp = container_of(p, struct jprobe, kp); - extern const char jprobe_return_break[]; - - if (instruction_pointer(regs) != (u64) jprobe_return_break) - return 0; - - if (orig_sp != stack_addr) { - struct pt_regs *saved_regs = - (struct pt_regs *)kcb->jprobe_saved_regs.sp; - pr_err("current sp %lx does not match saved sp %lx\n", - orig_sp, stack_addr); - pr_err("Saved registers for jprobe %p\n", jp); - __show_regs(saved_regs); - pr_err("Current registers\n"); - __show_regs(regs); - BUG(); - } - unpause_graph_tracing(); - *regs = kcb->jprobe_saved_regs; - preempt_enable_no_resched(); - return 1; -} - bool arch_within_kprobe_blacklist(unsigned long addr) { if ((addr >= (unsigned long)__kprobes_text_start && diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 137710f4dac3..68755fd70dcf 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ +lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_in_user.o copy_page.o \ clear_page.o memchr.o memcpy.o memmove.o memset.o \ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ diff --git a/arch/arm64/lib/bitops.S b/arch/arm64/lib/bitops.S deleted file mode 100644 index 43ac736baa5b..000000000000 --- a/arch/arm64/lib/bitops.S +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Based on arch/arm/lib/bitops.h - * - * Copyright (C) 2013 ARM Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/lse.h> - -/* - * x0: bits 5:0 bit offset - * bits 31:6 word offset - * x1: address - */ - .macro bitop, name, llsc, lse -ENTRY( \name ) - and w3, w0, #63 // Get bit offset - eor w0, w0, w3 // Clear low bits - mov x2, #1 - add x1, x1, x0, lsr #3 // Get word offset -alt_lse " prfm pstl1strm, [x1]", "nop" - lsl x3, x2, x3 // Create mask - -alt_lse "1: ldxr x2, [x1]", "\lse x3, [x1]" -alt_lse " \llsc x2, x2, x3", "nop" -alt_lse " stxr w0, x2, [x1]", "nop" -alt_lse " cbnz w0, 1b", "nop" - - ret -ENDPROC(\name ) - .endm - - .macro testop, name, llsc, lse -ENTRY( \name ) - and w3, w0, #63 // Get bit offset - eor w0, w0, w3 // Clear low bits - mov x2, #1 - add x1, x1, x0, lsr #3 // Get word offset -alt_lse " prfm pstl1strm, [x1]", "nop" - lsl x4, x2, x3 // Create mask - -alt_lse "1: ldxr x2, [x1]", "\lse x4, x2, [x1]" - lsr x0, x2, x3 -alt_lse " \llsc x2, x2, x4", "nop" -alt_lse " stlxr w5, x2, [x1]", "nop" -alt_lse " cbnz w5, 1b", "nop" -alt_lse " dmb ish", "nop" - - and x0, x0, #1 - ret -ENDPROC(\name ) - .endm - -/* - * Atomic bit operations. - */ - bitop change_bit, eor, steor - bitop clear_bit, bic, stclr - bitop set_bit, orr, stset - - testop test_and_change_bit, eor, ldeoral - testop test_and_clear_bit, bic, ldclral - testop test_and_set_bit, orr, ldsetal diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index ecc6818191df..192b3ba07075 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -108,7 +108,6 @@ static pte_t get_clear_flush(struct mm_struct *mm, unsigned long pgsize, unsigned long ncontig) { - struct vm_area_struct vma = { .vm_mm = mm }; pte_t orig_pte = huge_ptep_get(ptep); bool valid = pte_valid(orig_pte); unsigned long i, saddr = addr; @@ -125,8 +124,10 @@ static pte_t get_clear_flush(struct mm_struct *mm, orig_pte = pte_mkdirty(orig_pte); } - if (valid) + if (valid) { + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); flush_tlb_range(&vma, saddr, addr); + } return orig_pte; } @@ -145,7 +146,7 @@ static void clear_flush(struct mm_struct *mm, unsigned long pgsize, unsigned long ncontig) { - struct vm_area_struct vma = { .vm_mm = mm }; + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); unsigned long i, saddr = addr; for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 325cfb3b858a..9abf8a1e7b25 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -611,11 +611,13 @@ void __init mem_init(void) BUILD_BUG_ON(TASK_SIZE_32 > TASK_SIZE_64); #endif +#ifdef CONFIG_SPARSEMEM_VMEMMAP /* * Make sure we chose the upper bound of sizeof(struct page) - * correctly. + * correctly when sizing the VMEMMAP array. */ BUILD_BUG_ON(sizeof(struct page) > (1 << STRUCT_PAGE_MAX_SHIFT)); +#endif if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 493ff75670ff..8ae5d7ae4af3 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -977,12 +977,12 @@ int pmd_clear_huge(pmd_t *pmdp) return 1; } -int pud_free_pmd_page(pud_t *pud) +int pud_free_pmd_page(pud_t *pud, unsigned long addr) { return pud_none(*pud); } -int pmd_free_pte_page(pmd_t *pmd) +int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) { return pmd_none(*pmd); } diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index 941e7554e886..c6b6a06231b2 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -2,8 +2,10 @@ #ifndef __ARCH_H8300_ATOMIC__ #define __ARCH_H8300_ATOMIC__ +#include <linux/compiler.h> #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/irqflags.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -15,8 +17,6 @@ #define atomic_read(v) READ_ONCE((v)->counter) #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) -#include <linux/kernel.h> - #define ATOMIC_OP_RETURN(op, c_op) \ static inline int atomic_##op##_return(int i, atomic_t *v) \ { \ @@ -69,18 +69,6 @@ ATOMIC_OPS(sub, -=) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_inc_return(v) atomic_add_return(1, v) -#define atomic_dec_return(v) atomic_sub_return(1, v) - -#define atomic_inc(v) (void)atomic_inc_return(v) -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -#define atomic_dec(v) (void)atomic_dec_return(v) -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) - static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; @@ -94,7 +82,7 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int ret; h8300flags flags; @@ -106,5 +94,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) arch_local_irq_restore(flags); return ret; } +#define atomic_fetch_add_unless atomic_fetch_add_unless #endif /* __ARCH_H8300_ATOMIC __ */ diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index fb3dfb2a667e..311b9894ccc8 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -164,7 +164,7 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP /** - * __atomic_add_unless - add unless the number is a given value + * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer to value * @a: amount to add * @u: unless value is equal to u @@ -173,7 +173,7 @@ ATOMIC_OPS(xor) * */ -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int __oldval; register int tmp; @@ -196,18 +196,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) ); return __oldval; } - -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - -#define atomic_inc(v) atomic_add(1, (v)) -#define atomic_dec(v) atomic_sub(1, (v)) - -#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, (v)) == 0) -#define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0) - -#define atomic_inc_return(v) (atomic_add_return(1, v)) -#define atomic_dec_return(v) (atomic_sub_return(1, v)) +#define atomic_fetch_add_unless atomic_fetch_add_unless #endif diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 2524fb60fbc2..206530d0751b 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -215,91 +215,10 @@ ATOMIC64_FETCH_OP(xor, ^) (cmpxchg(&((v)->counter), old, new)) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - - -static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - c = atomic64_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic64_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c != (u); -} - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -static __inline__ long atomic64_dec_if_positive(atomic64_t *v) -{ - long c, old, dec; - c = atomic64_read(v); - for (;;) { - dec = c - 1; - if (unlikely(dec < 0)) - break; - old = atomic64_cmpxchg((v), c, dec); - if (likely(old == c)) - break; - c = old; - } - return dec; -} - -/* - * Atomically add I to V and return TRUE if the resulting value is - * negative. - */ -static __inline__ int -atomic_add_negative (int i, atomic_t *v) -{ - return atomic_add_return(i, v) < 0; -} - -static __inline__ long -atomic64_add_negative (__s64 i, atomic64_t *v) -{ - return atomic64_add_return(i, v) < 0; -} - -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) -#define atomic64_dec_return(v) atomic64_sub_return(1, (v)) -#define atomic64_inc_return(v) atomic64_add_return(1, (v)) - -#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) -#define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) -#define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0) -#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) -#define atomic64_inc_and_test(v) (atomic64_add_return(1, (v)) == 0) - #define atomic_add(i,v) (void)atomic_add_return((i), (v)) #define atomic_sub(i,v) (void)atomic_sub_return((i), (v)) -#define atomic_inc(v) atomic_add(1, (v)) -#define atomic_dec(v) atomic_sub(1, (v)) #define atomic64_add(i,v) (void)atomic64_add_return((i), (v)) #define atomic64_sub(i,v) (void)atomic64_sub_return((i), (v)) -#define atomic64_inc(v) atomic64_add(1, (v)) -#define atomic64_dec(v) atomic64_sub(1, (v)) #endif /* _ASM_IA64_ATOMIC_H */ diff --git a/arch/ia64/include/asm/kprobes.h b/arch/ia64/include/asm/kprobes.h index 0302b3664789..580356a2eea6 100644 --- a/arch/ia64/include/asm/kprobes.h +++ b/arch/ia64/include/asm/kprobes.h @@ -82,8 +82,6 @@ struct prev_kprobe { #define ARCH_PREV_KPROBE_SZ 2 struct kprobe_ctlblk { unsigned long kprobe_status; - struct pt_regs jprobe_saved_regs; - unsigned long jprobes_saved_stacked_regs[MAX_PARAM_RSE_SIZE]; unsigned long *bsp; unsigned long cfm; atomic_t prev_kprobe_index; diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 44f0ac0df308..516355a774bf 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -115,12 +115,11 @@ ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned flush_tlb_all(); } else { /* - * XXX fix me: flush_tlb_range() should take an mm pointer instead of a - * vma pointer. + * flush_tlb_range() takes a vma instead of a mm pointer because + * some architectures want the vm_flags for ITLB/DTLB flush. */ - struct vm_area_struct vma; + struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0); - vma.vm_mm = tlb->mm; /* flush the address range from the tlb: */ flush_tlb_range(&vma, start, end); /* now flush the virt. page-table area mapping the address range: */ diff --git a/arch/ia64/include/uapi/asm/break.h b/arch/ia64/include/uapi/asm/break.h index 5d742bcb0018..4ca110f0a94b 100644 --- a/arch/ia64/include/uapi/asm/break.h +++ b/arch/ia64/include/uapi/asm/break.h @@ -14,7 +14,6 @@ */ #define __IA64_BREAK_KDB 0x80100 #define __IA64_BREAK_KPROBE 0x81000 /* .. 0x81fff */ -#define __IA64_BREAK_JPROBE 0x82000 /* * OS-specific break numbers: diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index 498f3da3f225..d0c0ccdd656a 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -25,7 +25,7 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_PERFMON) += perfmon_default_smpl.o obj-$(CONFIG_IA64_CYCLONE) += cyclone.o obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o -obj-$(CONFIG_KPROBES) += kprobes.o jprobes.o +obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/ia64/kernel/jprobes.S b/arch/ia64/kernel/jprobes.S deleted file mode 100644 index f69389c7be1d..000000000000 --- a/arch/ia64/kernel/jprobes.S +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Jprobe specific operations - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * Copyright (C) Intel Corporation, 2005 - * - * 2005-May Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy - * <anil.s.keshavamurthy@intel.com> initial implementation - * - * Jprobes (a.k.a. "jump probes" which is built on-top of kprobes) allow a - * probe to be inserted into the beginning of a function call. The fundamental - * difference between a jprobe and a kprobe is the jprobe handler is executed - * in the same context as the target function, while the kprobe handlers - * are executed in interrupt context. - * - * For jprobes we initially gain control by placing a break point in the - * first instruction of the targeted function. When we catch that specific - * break, we: - * * set the return address to our jprobe_inst_return() function - * * jump to the jprobe handler function - * - * Since we fixed up the return address, the jprobe handler will return to our - * jprobe_inst_return() function, giving us control again. At this point we - * are back in the parents frame marker, so we do yet another call to our - * jprobe_break() function to fix up the frame marker as it would normally - * exist in the target function. - * - * Our jprobe_return function then transfers control back to kprobes.c by - * executing a break instruction using one of our reserved numbers. When we - * catch that break in kprobes.c, we continue like we do for a normal kprobe - * by single stepping the emulated instruction, and then returning execution - * to the correct location. - */ -#include <asm/asmmacro.h> -#include <asm/break.h> - - /* - * void jprobe_break(void) - */ - .section .kprobes.text, "ax" -ENTRY(jprobe_break) - break.m __IA64_BREAK_JPROBE -END(jprobe_break) - - /* - * void jprobe_inst_return(void) - */ -GLOBAL_ENTRY(jprobe_inst_return) - br.call.sptk.many b0=jprobe_break -END(jprobe_inst_return) - -GLOBAL_ENTRY(invalidate_stacked_regs) - movl r16=invalidate_restore_cfm - ;; - mov b6=r16 - ;; - br.ret.sptk.many b6 - ;; -invalidate_restore_cfm: - mov r16=ar.rsc - ;; - mov ar.rsc=r0 - ;; - loadrs - ;; - mov ar.rsc=r16 - ;; - br.cond.sptk.many rp -END(invalidate_stacked_regs) - -GLOBAL_ENTRY(flush_register_stack) - // flush dirty regs to backing store (must be first in insn group) - flushrs - ;; - br.ret.sptk.many rp -END(flush_register_stack) - diff --git a/arch/ia64/kernel/kprobes.c b/arch/ia64/kernel/kprobes.c index f5f3a5e6fcd1..aa41bd5cf9b7 100644 --- a/arch/ia64/kernel/kprobes.c +++ b/arch/ia64/kernel/kprobes.c @@ -35,8 +35,6 @@ #include <asm/sections.h> #include <asm/exception.h> -extern void jprobe_inst_return(void); - DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); @@ -480,12 +478,9 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) */ break; } - kretprobe_assert(ri, orig_ret_address, trampoline_address); - reset_current_kprobe(); kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); @@ -819,14 +814,6 @@ static int __kprobes pre_kprobes_handler(struct die_args *args) prepare_ss(p, regs); kcb->kprobe_status = KPROBE_REENTER; return 1; - } else if (args->err == __IA64_BREAK_JPROBE) { - /* - * jprobe instrumented function just completed - */ - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - goto ss_probe; - } } else if (!is_ia64_break_inst(regs)) { /* The breakpoint instruction was removed by * another cpu right after we hit, no further @@ -861,15 +848,12 @@ static int __kprobes pre_kprobes_handler(struct die_args *args) set_current_kprobe(p, kcb); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (p->pre_handler && p->pre_handler(p, regs)) - /* - * Our pre-handler is specifically requesting that we just - * do a return. This is used for both the jprobe pre-handler - * and the kretprobe trampoline - */ + if (p->pre_handler && p->pre_handler(p, regs)) { + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; + } -ss_probe: #if !defined(CONFIG_PREEMPT) if (p->ainsn.inst_flag == INST_FLAG_BOOSTABLE && !p->post_handler) { /* Boost up -- we can execute copied instructions directly */ @@ -992,7 +976,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, case DIE_BREAK: /* err is break number from ia64_bad_break() */ if ((args->err >> 12) == (__IA64_BREAK_KPROBE >> 12) - || args->err == __IA64_BREAK_JPROBE || args->err == 0) if (pre_kprobes_handler(args)) ret = NOTIFY_STOP; @@ -1040,74 +1023,6 @@ unsigned long arch_deref_entry_point(void *entry) return ((struct fnptr *)entry)->ip; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - unsigned long addr = arch_deref_entry_point(jp->entry); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - struct param_bsp_cfm pa; - int bytes; - - /* - * Callee owns the argument space and could overwrite it, eg - * tail call optimization. So to be absolutely safe - * we save the argument space before transferring the control - * to instrumented jprobe function which runs in - * the process context - */ - pa.ip = regs->cr_iip; - unw_init_running(ia64_get_bsp_cfm, &pa); - bytes = (char *)ia64_rse_skip_regs(pa.bsp, pa.cfm & 0x3f) - - (char *)pa.bsp; - memcpy( kcb->jprobes_saved_stacked_regs, - pa.bsp, - bytes ); - kcb->bsp = pa.bsp; - kcb->cfm = pa.cfm; - - /* save architectural state */ - kcb->jprobe_saved_regs = *regs; - - /* after rfi, execute the jprobe instrumented function */ - regs->cr_iip = addr & ~0xFULL; - ia64_psr(regs)->ri = addr & 0xf; - regs->r1 = ((struct fnptr *)(jp->entry))->gp; - - /* - * fix the return address to our jprobe_inst_return() function - * in the jprobes.S file - */ - regs->b0 = ((struct fnptr *)(jprobe_inst_return))->ip; - - return 1; -} - -/* ia64 does not need this */ -void __kprobes jprobe_return(void) -{ -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - int bytes; - - /* restoring architectural state */ - *regs = kcb->jprobe_saved_regs; - - /* restoring the original argument space */ - flush_register_stack(); - bytes = (char *)ia64_rse_skip_regs(kcb->bsp, kcb->cfm & 0x3f) - - (char *)kcb->bsp; - memcpy( kcb->bsp, - kcb->jprobes_saved_stacked_regs, - bytes ); - invalidate_stacked_regs(); - - preempt_enable_no_resched(); - return 1; -} - static struct kprobe trampoline_p = { .pre_handler = trampoline_probe_handler }; diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c index 3b38c717008a..46bff1661836 100644 --- a/arch/ia64/kernel/perfmon.c +++ b/arch/ia64/kernel/perfmon.c @@ -2278,17 +2278,15 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t DPRINT(("smpl_buf @%p\n", smpl_buf)); /* allocate vma */ - vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); + vma = vm_area_alloc(mm); if (!vma) { DPRINT(("Cannot allocate vma\n")); goto error_kmem; } - INIT_LIST_HEAD(&vma->anon_vma_chain); /* * partially initialize the vma for the sampling buffer */ - vma->vm_mm = mm; vma->vm_file = get_file(filp); vma->vm_flags = VM_READ|VM_MAYREAD|VM_DONTEXPAND|VM_DONTDUMP; vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */ @@ -2346,7 +2344,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t return 0; error: - kmem_cache_free(vm_area_cachep, vma); + vm_area_free(vma); error_kmem: pfm_rvfree(smpl_buf, size); diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 18278b448530..3b85c3ecac38 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -114,10 +114,9 @@ ia64_init_addr_space (void) * the problem. When the process attempts to write to the register backing store * for the first time, it will get a SEGFAULT in this case. */ - vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); + vma = vm_area_alloc(current->mm); if (vma) { - INIT_LIST_HEAD(&vma->anon_vma_chain); - vma->vm_mm = current->mm; + vma_set_anonymous(vma); vma->vm_start = current->thread.rbs_bot & PAGE_MASK; vma->vm_end = vma->vm_start + PAGE_SIZE; vma->vm_flags = VM_DATA_DEFAULT_FLAGS|VM_GROWSUP|VM_ACCOUNT; @@ -125,7 +124,7 @@ ia64_init_addr_space (void) down_write(¤t->mm->mmap_sem); if (insert_vm_struct(current->mm, vma)) { up_write(¤t->mm->mmap_sem); - kmem_cache_free(vm_area_cachep, vma); + vm_area_free(vma); return; } up_write(¤t->mm->mmap_sem); @@ -133,10 +132,9 @@ ia64_init_addr_space (void) /* map NaT-page at address zero to speed up speculative dereferencing of NULL: */ if (!(current->personality & MMAP_PAGE_ZERO)) { - vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); + vma = vm_area_alloc(current->mm); if (vma) { - INIT_LIST_HEAD(&vma->anon_vma_chain); - vma->vm_mm = current->mm; + vma_set_anonymous(vma); vma->vm_end = PAGE_SIZE; vma->vm_page_prot = __pgprot(pgprot_val(PAGE_READONLY) | _PAGE_MA_NAT); vma->vm_flags = VM_READ | VM_MAYREAD | VM_IO | @@ -144,7 +142,7 @@ ia64_init_addr_space (void) down_write(¤t->mm->mmap_sem); if (insert_vm_struct(current->mm, vma)) { up_write(¤t->mm->mmap_sem); - kmem_cache_free(vm_area_cachep, vma); + vm_area_free(vma); return; } up_write(¤t->mm->mmap_sem); @@ -277,7 +275,7 @@ static struct vm_area_struct gate_vma; static int __init gate_vma_init(void) { - gate_vma.vm_mm = NULL; + vma_init(&gate_vma, NULL); gate_vma.vm_start = FIXADDR_USER_START; gate_vma.vm_end = FIXADDR_USER_END; gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 785612b576f7..b29f93774d95 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -2,6 +2,7 @@ config M68K bool default y + select ARCH_HAS_SYNC_DMA_FOR_DEVICE if HAS_DMA select ARCH_MIGHT_HAVE_PC_PARPORT if ISA select ARCH_NO_COHERENT_DMA_MMAP if !MMU select HAVE_IDE @@ -24,6 +25,10 @@ config M68K select MODULES_USE_ELF_RELA select OLD_SIGSUSPEND3 select OLD_SIGACTION + select DMA_NONCOHERENT_OPS if HAS_DMA + select HAVE_MEMBLOCK + select ARCH_DISCARD_MEMBLOCK + select NO_BOOTMEM config CPU_BIG_ENDIAN def_bool y diff --git a/arch/m68k/apollo/config.c b/arch/m68k/apollo/config.c index b2a6bc63f8cd..aef8d42e078d 100644 --- a/arch/m68k/apollo/config.c +++ b/arch/m68k/apollo/config.c @@ -31,7 +31,6 @@ extern void dn_sched_init(irq_handler_t handler); extern void dn_init_IRQ(void); extern u32 dn_gettimeoffset(void); extern int dn_dummy_hwclk(int, struct rtc_time *); -extern int dn_dummy_set_clock_mmss(unsigned long); extern void dn_dummy_reset(void); #ifdef CONFIG_HEARTBEAT static void dn_heartbeat(int on); @@ -156,7 +155,6 @@ void __init config_apollo(void) arch_gettimeoffset = dn_gettimeoffset; mach_max_dma_address = 0xffffffff; mach_hwclk = dn_dummy_hwclk; /* */ - mach_set_clock_mmss = dn_dummy_set_clock_mmss; /* */ mach_reset = dn_dummy_reset; /* */ #ifdef CONFIG_HEARTBEAT mach_heartbeat = dn_heartbeat; @@ -240,12 +238,6 @@ int dn_dummy_hwclk(int op, struct rtc_time *t) { } -int dn_dummy_set_clock_mmss(unsigned long nowtime) -{ - pr_info("set_clock_mmss\n"); - return 0; -} - void dn_dummy_reset(void) { dn_serial_print("The end !\n"); diff --git a/arch/m68k/atari/config.c b/arch/m68k/atari/config.c index 565c6f06ab0b..bd96702a1ad0 100644 --- a/arch/m68k/atari/config.c +++ b/arch/m68k/atari/config.c @@ -81,9 +81,6 @@ extern void atari_sched_init(irq_handler_t); extern u32 atari_gettimeoffset(void); extern int atari_mste_hwclk (int, struct rtc_time *); extern int atari_tt_hwclk (int, struct rtc_time *); -extern int atari_mste_set_clock_mmss (unsigned long); -extern int atari_tt_set_clock_mmss (unsigned long); - /* ++roman: This is a more elaborate test for an SCC chip, since the plain * Medusa board generates DTACK at the SCC's standard addresses, but a SCC @@ -362,13 +359,11 @@ void __init config_atari(void) ATARIHW_SET(TT_CLK); pr_cont(" TT_CLK"); mach_hwclk = atari_tt_hwclk; - mach_set_clock_mmss = atari_tt_set_clock_mmss; } if (hwreg_present(&mste_rtc.sec_ones)) { ATARIHW_SET(MSTE_CLK); pr_cont(" MSTE_CLK"); mach_hwclk = atari_mste_hwclk; - mach_set_clock_mmss = atari_mste_set_clock_mmss; } if (!MACH_IS_MEDUSA && hwreg_present(&dma_wd.fdc_speed) && hwreg_write(&dma_wd.fdc_speed, 0)) { diff --git a/arch/m68k/atari/time.c b/arch/m68k/atari/time.c index c549b48174ec..9cca64286464 100644 --- a/arch/m68k/atari/time.c +++ b/arch/m68k/atari/time.c @@ -285,69 +285,6 @@ int atari_tt_hwclk( int op, struct rtc_time *t ) return( 0 ); } - -int atari_mste_set_clock_mmss (unsigned long nowtime) -{ - short real_seconds = nowtime % 60, real_minutes = (nowtime / 60) % 60; - struct MSTE_RTC val; - unsigned char rtc_minutes; - - mste_read(&val); - rtc_minutes= val.min_ones + val.min_tens * 10; - if ((rtc_minutes < real_minutes - ? real_minutes - rtc_minutes - : rtc_minutes - real_minutes) < 30) - { - val.sec_ones = real_seconds % 10; - val.sec_tens = real_seconds / 10; - val.min_ones = real_minutes % 10; - val.min_tens = real_minutes / 10; - mste_write(&val); - } - else - return -1; - return 0; -} - -int atari_tt_set_clock_mmss (unsigned long nowtime) -{ - int retval = 0; - short real_seconds = nowtime % 60, real_minutes = (nowtime / 60) % 60; - unsigned char save_control, save_freq_select, rtc_minutes; - - save_control = RTC_READ (RTC_CONTROL); /* tell the clock it's being set */ - RTC_WRITE (RTC_CONTROL, save_control | RTC_SET); - - save_freq_select = RTC_READ (RTC_FREQ_SELECT); /* stop and reset prescaler */ - RTC_WRITE (RTC_FREQ_SELECT, save_freq_select | RTC_DIV_RESET2); - - rtc_minutes = RTC_READ (RTC_MINUTES); - if (!(save_control & RTC_DM_BINARY)) - rtc_minutes = bcd2bin(rtc_minutes); - - /* Since we're only adjusting minutes and seconds, don't interfere - with hour overflow. This avoids messing with unknown time zones - but requires your RTC not to be off by more than 30 minutes. */ - if ((rtc_minutes < real_minutes - ? real_minutes - rtc_minutes - : rtc_minutes - real_minutes) < 30) - { - if (!(save_control & RTC_DM_BINARY)) - { - real_seconds = bin2bcd(real_seconds); - real_minutes = bin2bcd(real_minutes); - } - RTC_WRITE (RTC_SECONDS, real_seconds); - RTC_WRITE (RTC_MINUTES, real_minutes); - } - else - retval = -1; - - RTC_WRITE (RTC_FREQ_SELECT, save_freq_select); - RTC_WRITE (RTC_CONTROL, save_control); - return retval; -} - /* * Local variables: * c-indent-level: 4 diff --git a/arch/m68k/bvme6000/config.c b/arch/m68k/bvme6000/config.c index 2cfff4765040..143ee9fa3893 100644 --- a/arch/m68k/bvme6000/config.c +++ b/arch/m68k/bvme6000/config.c @@ -41,7 +41,6 @@ static void bvme6000_get_model(char *model); extern void bvme6000_sched_init(irq_handler_t handler); extern u32 bvme6000_gettimeoffset(void); extern int bvme6000_hwclk (int, struct rtc_time *); -extern int bvme6000_set_clock_mmss (unsigned long); extern void bvme6000_reset (void); void bvme6000_set_vectors (void); @@ -113,7 +112,6 @@ void __init config_bvme6000(void) mach_init_IRQ = bvme6000_init_IRQ; arch_gettimeoffset = bvme6000_gettimeoffset; mach_hwclk = bvme6000_hwclk; - mach_set_clock_mmss = bvme6000_set_clock_mmss; mach_reset = bvme6000_reset; mach_get_model = bvme6000_get_model; @@ -305,46 +303,3 @@ int bvme6000_hwclk(int op, struct rtc_time *t) return 0; } - -/* - * Set the minutes and seconds from seconds value 'nowtime'. Fail if - * clock is out by > 30 minutes. Logic lifted from atari code. - * Algorithm is to wait for the 10ms register to change, and then to - * wait a short while, and then set it. - */ - -int bvme6000_set_clock_mmss (unsigned long nowtime) -{ - int retval = 0; - short real_seconds = nowtime % 60, real_minutes = (nowtime / 60) % 60; - unsigned char rtc_minutes, rtc_tenms; - volatile RtcPtr_t rtc = (RtcPtr_t)BVME_RTC_BASE; - unsigned char msr = rtc->msr & 0xc0; - unsigned long flags; - volatile int i; - - rtc->msr = 0; /* Ensure clock accessible */ - rtc_minutes = bcd2bin (rtc->bcd_min); - - if ((rtc_minutes < real_minutes - ? real_minutes - rtc_minutes - : rtc_minutes - real_minutes) < 30) - { - local_irq_save(flags); - rtc_tenms = rtc->bcd_tenms; - while (rtc_tenms == rtc->bcd_tenms) - ; - for (i = 0; i < 1000; i++) - ; - rtc->bcd_min = bin2bcd(real_minutes); - rtc->bcd_sec = bin2bcd(real_seconds); - local_irq_restore(flags); - } - else - retval = -1; - - rtc->msr = msr; - - return retval; -} - diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index a874e54404d1..1d5483f6e457 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -52,6 +52,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -98,18 +99,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -122,6 +119,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -200,7 +198,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -231,7 +228,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -260,7 +256,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -301,6 +296,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -356,6 +352,7 @@ CONFIG_A2091_SCSI=y CONFIG_GVP11_SCSI=y CONFIG_SCSI_A4000T=y CONFIG_SCSI_ZORRO7XX=y +CONFIG_SCSI_ZORRO_ESP=y CONFIG_MD=y CONFIG_MD_LINEAR=m CONFIG_BLK_DEV_DM=m @@ -363,6 +360,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -402,8 +400,8 @@ CONFIG_A2065=y CONFIG_ARIADNE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CIRRUS is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set @@ -412,8 +410,10 @@ CONFIG_ARIADNE=y # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set +CONFIG_XSURF100=y CONFIG_HYDRA=y CONFIG_APNE=y CONFIG_ZORRO8390=y @@ -426,9 +426,9 @@ CONFIG_ZORRO8390=y # CONFIG_NET_VENDOR_SMSC is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -478,6 +478,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -499,7 +500,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -600,6 +601,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -622,6 +624,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -657,6 +664,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 8ce39e23aa42..52a0af127951 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -50,6 +50,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -96,18 +97,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -120,6 +117,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -198,7 +196,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -229,7 +226,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -258,7 +254,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -299,6 +294,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -345,6 +341,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -381,14 +378,15 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_AMAZON is not set # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -400,9 +398,9 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -440,6 +438,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -458,7 +457,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -559,6 +558,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -581,6 +581,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -616,6 +621,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 346c4e75edf8..b3103e51268a 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -50,6 +50,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -96,18 +97,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -120,6 +117,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -198,7 +196,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -229,7 +226,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -258,7 +254,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -299,6 +294,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -354,6 +350,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -391,14 +388,15 @@ CONFIG_VETH=m CONFIG_ATARILANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set CONFIG_NE2000=y @@ -411,9 +409,9 @@ CONFIG_NE2000=y CONFIG_SMC91X=y # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -480,7 +478,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -581,6 +579,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -603,6 +602,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -638,6 +642,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index fca9c7aa71a3..fb7d651a4cab 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -48,6 +48,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -94,18 +95,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -118,6 +115,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -196,7 +194,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -227,7 +224,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -256,7 +252,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -297,6 +292,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -344,6 +340,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -380,14 +377,15 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_AMAZON is not set # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set CONFIG_BVME6000_NET=y # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -399,9 +397,9 @@ CONFIG_BVME6000_NET=y # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -433,6 +431,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -450,7 +449,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -551,6 +550,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -573,6 +573,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -608,6 +613,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index f9eab174915c..6b37f5537c39 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -50,6 +50,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -96,18 +97,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -120,6 +117,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -198,7 +196,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -229,7 +226,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -258,7 +254,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -299,6 +294,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -345,6 +341,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -382,14 +379,15 @@ CONFIG_VETH=m CONFIG_HPLANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -401,9 +399,9 @@ CONFIG_HPLANCE=y # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -443,6 +441,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -460,7 +459,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -561,6 +560,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -583,6 +583,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -618,6 +623,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index b52e597899eb..930cc2965a11 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -49,6 +49,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -95,18 +96,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -119,6 +116,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -197,7 +195,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -228,7 +225,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -257,7 +253,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -301,6 +296,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -354,6 +350,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -398,8 +395,8 @@ CONFIG_VETH=m CONFIG_MACMACE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set CONFIG_MAC89x0=y # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set @@ -407,6 +404,7 @@ CONFIG_MAC89x0=y # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set CONFIG_MACSONIC=y # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -420,9 +418,9 @@ CONFIG_MAC8390=y # CONFIG_NET_VENDOR_SMSC is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -465,6 +463,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -482,7 +481,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -583,6 +582,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -605,6 +605,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -640,6 +645,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 2a84eeec5b02..e7dd25300127 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -59,6 +59,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -105,18 +106,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -129,6 +126,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -207,7 +205,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -238,7 +235,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -267,7 +263,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -311,6 +306,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -373,6 +369,7 @@ CONFIG_A2091_SCSI=y CONFIG_GVP11_SCSI=y CONFIG_SCSI_A4000T=y CONFIG_SCSI_ZORRO7XX=y +CONFIG_SCSI_ZORRO_ESP=y CONFIG_ATARI_SCSI=y CONFIG_MAC_SCSI=y CONFIG_SCSI_MAC_ESP=y @@ -387,6 +384,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -438,8 +436,8 @@ CONFIG_SUN3LANCE=y CONFIG_MACMACE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set CONFIG_MAC89x0=y # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set @@ -449,9 +447,11 @@ CONFIG_BVME6000_NET=y CONFIG_MVME16x_NET=y # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set CONFIG_MACSONIC=y # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set +CONFIG_XSURF100=y CONFIG_HYDRA=y CONFIG_MAC8390=y CONFIG_NE2000=y @@ -466,9 +466,9 @@ CONFIG_ZORRO8390=y CONFIG_SMC91X=y # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PLIP=m CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m @@ -533,6 +533,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -562,7 +563,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -663,6 +664,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -685,6 +687,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -720,6 +727,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 476e69994340..b383327fd77a 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -47,6 +47,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -93,18 +94,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -117,6 +114,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -195,7 +193,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -226,7 +223,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -255,7 +251,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -296,6 +291,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -343,6 +339,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -380,14 +377,15 @@ CONFIG_VETH=m CONFIG_MVME147_NET=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -399,9 +397,9 @@ CONFIG_MVME147_NET=y # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -433,6 +431,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -450,7 +449,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -551,6 +550,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -573,6 +573,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -608,6 +613,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 1477cda9146e..9783d3deb9e9 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -48,6 +48,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -94,18 +95,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -118,6 +115,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -196,7 +194,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -227,7 +224,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -256,7 +252,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -297,6 +292,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -344,6 +340,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -380,14 +377,15 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_AMAZON is not set # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set CONFIG_MVME16x_NET=y # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -399,9 +397,9 @@ CONFIG_MVME16x_NET=y # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -433,6 +431,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -450,7 +449,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -551,6 +550,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -573,6 +573,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -608,6 +613,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index b3a543dc48a0..a35d10ee10cb 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -48,6 +48,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -94,18 +95,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -118,6 +115,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -196,7 +194,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -227,7 +224,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -256,7 +252,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -297,6 +292,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -350,6 +346,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -388,8 +385,8 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_AMD is not set # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CIRRUS is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set @@ -398,6 +395,7 @@ CONFIG_VETH=m # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set CONFIG_NE2000=y @@ -410,9 +408,9 @@ CONFIG_NE2000=y # CONFIG_NET_VENDOR_SMSC is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PLIP=m CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m @@ -455,6 +453,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -473,7 +472,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -574,6 +573,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -596,6 +596,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -631,6 +636,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index d543ed5dfa96..573bf922d448 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -45,6 +45,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -91,18 +92,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -115,6 +112,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -193,7 +191,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -224,7 +221,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -253,7 +249,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -294,6 +289,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -341,6 +337,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -385,6 +382,7 @@ CONFIG_SUN3LANCE=y CONFIG_SUN3_82586=y # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -397,9 +395,9 @@ CONFIG_SUN3_82586=y # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set # CONFIG_NET_VENDOR_SUN is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -435,6 +433,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -452,7 +451,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -553,6 +552,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -574,6 +574,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -609,6 +614,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index a67e54246023..efb27a7fcc55 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -45,6 +45,7 @@ CONFIG_UNIX_DIAG=m CONFIG_TLS=m CONFIG_XFRM_MIGRATE=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y @@ -91,18 +92,14 @@ CONFIG_NF_CONNTRACK_SANE=m CONFIG_NF_CONNTRACK_SIP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_SET=m CONFIG_NF_TABLES_INET=y CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_EXTHDR=m -CONFIG_NFT_META=m -CONFIG_NFT_RT=m CONFIG_NFT_NUMGEN=m CONFIG_NFT_CT=m CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_SET_RBTREE=m -CONFIG_NFT_SET_HASH=m -CONFIG_NFT_SET_BITMAP=m CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m CONFIG_NFT_MASQ=m @@ -115,6 +112,7 @@ CONFIG_NFT_REJECT=m CONFIG_NFT_COMPAT=m CONFIG_NFT_HASH=m CONFIG_NFT_FIB_INET=m +CONFIG_NFT_SOCKET=m CONFIG_NFT_DUP_NETDEV=m CONFIG_NFT_FWD_NETDEV=m CONFIG_NFT_FIB_NETDEV=m @@ -193,7 +191,6 @@ CONFIG_IP_SET_HASH_NETPORT=m CONFIG_IP_SET_HASH_NETIFACE=m CONFIG_IP_SET_LIST_SET=m CONFIG_NF_CONNTRACK_IPV4=m -CONFIG_NF_SOCKET_IPV4=m CONFIG_NFT_CHAIN_ROUTE_IPV4=m CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m @@ -224,7 +221,6 @@ CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m CONFIG_NF_CONNTRACK_IPV6=m -CONFIG_NF_SOCKET_IPV6=m CONFIG_NFT_CHAIN_ROUTE_IPV6=m CONFIG_NFT_CHAIN_NAT_IPV6=m CONFIG_NFT_MASQ_IPV6=m @@ -253,7 +249,6 @@ CONFIG_IP6_NF_NAT=m CONFIG_IP6_NF_TARGET_MASQUERADE=m CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=y -CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m CONFIG_NF_LOG_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m @@ -294,6 +289,7 @@ CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m CONFIG_DNS_RESOLVER=y CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set CONFIG_BATMAN_ADV_DAT=y CONFIG_BATMAN_ADV_NC=y CONFIG_BATMAN_ADV_MCAST=y @@ -341,6 +337,7 @@ CONFIG_DM_UNSTRIPED=m CONFIG_DM_CRYPT=m CONFIG_DM_SNAPSHOT=m CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_WRITECACHE=m CONFIG_DM_ERA=m CONFIG_DM_MIRROR=m CONFIG_DM_RAID=m @@ -378,14 +375,15 @@ CONFIG_VETH=m CONFIG_SUN3LANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_CORTINA is not set # CONFIG_NET_VENDOR_EZCHIP is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROSEMI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETRONOME is not set # CONFIG_NET_VENDOR_NI is not set @@ -397,9 +395,9 @@ CONFIG_SUN3LANCE=y # CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set -# CONFIG_NET_VENDOR_SYNOPSYS is not set CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -435,6 +433,7 @@ CONFIG_HIDRAW=y CONFIG_UHID=m # CONFIG_HID_GENERIC is not set # CONFIG_HID_ITE is not set +# CONFIG_HID_REDRAGON is not set # CONFIG_USB_SUPPORT is not set CONFIG_RTC_CLASS=y # CONFIG_RTC_NVMEM is not set @@ -452,7 +451,7 @@ CONFIG_FS_ENCRYPTION=m CONFIG_FANOTIFY=y CONFIG_QUOTA_NETLINK_INTERFACE=y # CONFIG_PRINT_QUOTA_WARNING is not set -CONFIG_AUTOFS4_FS=m +CONFIG_AUTOFS_FS=m CONFIG_FUSE_FS=m CONFIG_CUSE=m CONFIG_OVERLAY_FS=m @@ -553,6 +552,7 @@ CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m CONFIG_TEST_UUID=m +CONFIG_TEST_OVERFLOW=m CONFIG_TEST_RHASHTABLE=m CONFIG_TEST_HASH=m CONFIG_TEST_USER_COPY=m @@ -575,6 +575,11 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_MCRYPTD=m CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128L=m +CONFIG_CRYPTO_AEGIS256=m +CONFIG_CRYPTO_MORUS640=m +CONFIG_CRYPTO_MORUS1280=m CONFIG_CRYPTO_CFB=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m @@ -610,6 +615,7 @@ CONFIG_CRYPTO_LZO=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ZSTD=m CONFIG_CRYPTO_ANSI_CPRNG=m CONFIG_CRYPTO_DRBG_HASH=y CONFIG_CRYPTO_DRBG_CTR=y diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index 4d8d68c4e3dd..a4b8d3331a9e 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild @@ -1,6 +1,7 @@ generic-y += barrier.h generic-y += compat.h generic-y += device.h +generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += exec.h generic-y += extable.h diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index e993e2860ee1..47228b0d4163 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -126,11 +126,13 @@ static inline void atomic_inc(atomic_t *v) { __asm__ __volatile__("addql #1,%0" : "+m" (*v)); } +#define atomic_inc atomic_inc static inline void atomic_dec(atomic_t *v) { __asm__ __volatile__("subql #1,%0" : "+m" (*v)); } +#define atomic_dec atomic_dec static inline int atomic_dec_and_test(atomic_t *v) { @@ -138,6 +140,7 @@ static inline int atomic_dec_and_test(atomic_t *v) __asm__ __volatile__("subql #1,%1; seq %0" : "=d" (c), "+m" (*v)); return c != 0; } +#define atomic_dec_and_test atomic_dec_and_test static inline int atomic_dec_and_test_lt(atomic_t *v) { @@ -155,6 +158,7 @@ static inline int atomic_inc_and_test(atomic_t *v) __asm__ __volatile__("addql #1,%1; seq %0" : "=d" (c), "+m" (*v)); return c != 0; } +#define atomic_inc_and_test atomic_inc_and_test #ifdef CONFIG_RMW_INSNS @@ -190,9 +194,6 @@ static inline int atomic_xchg(atomic_t *v, int new) #endif /* !CONFIG_RMW_INSNS */ -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) - static inline int atomic_sub_and_test(int i, atomic_t *v) { char c; @@ -201,6 +202,7 @@ static inline int atomic_sub_and_test(int i, atomic_t *v) : ASM_DI (i)); return c != 0; } +#define atomic_sub_and_test atomic_sub_and_test static inline int atomic_add_negative(int i, atomic_t *v) { @@ -210,20 +212,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) : ASM_DI (i)); return c != 0; } - -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} +#define atomic_add_negative atomic_add_negative #endif /* __ARCH_M68K_ATOMIC __ */ diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 93b47b1f6fb4..d979f38af751 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -454,7 +454,7 @@ static inline unsigned long ffz(unsigned long word) */ #if (defined(__mcfisaaplus__) || defined(__mcfisac__)) && \ !defined(CONFIG_M68000) && !defined(CONFIG_MCPU32) -static inline int __ffs(int x) +static inline unsigned long __ffs(unsigned long x) { __asm__ __volatile__ ("bitrev %0; ff1 %0" : "=d" (x) @@ -493,7 +493,11 @@ static inline int ffs(int x) : "dm" (x & -x)); return 32 - cnt; } -#define __ffs(x) (ffs(x) - 1) + +static inline unsigned long __ffs(unsigned long x) +{ + return ffs(x) - 1; +} /* * fls: find last bit set. @@ -515,12 +519,16 @@ static inline int __fls(int x) #endif +/* Simple test-and-set bit locks */ +#define test_and_set_bit_lock test_and_set_bit +#define clear_bit_unlock clear_bit +#define __clear_bit_unlock clear_bit_unlock + #include <asm-generic/bitops/ext2-atomic.h> #include <asm-generic/bitops/le.h> #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/sched.h> #include <asm-generic/bitops/hweight.h> -#include <asm-generic/bitops/lock.h> #endif /* __KERNEL__ */ #endif /* _M68K_BITOPS_H */ diff --git a/arch/m68k/include/asm/dma-mapping.h b/arch/m68k/include/asm/dma-mapping.h deleted file mode 100644 index e3722ed04fbb..000000000000 --- a/arch/m68k/include/asm/dma-mapping.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _M68K_DMA_MAPPING_H -#define _M68K_DMA_MAPPING_H - -extern const struct dma_map_ops m68k_dma_ops; - -static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) -{ - return &m68k_dma_ops; -} - -#endif /* _M68K_DMA_MAPPING_H */ diff --git a/arch/m68k/include/asm/io.h b/arch/m68k/include/asm/io.h index ca2849afb087..aabe6420ead2 100644 --- a/arch/m68k/include/asm/io.h +++ b/arch/m68k/include/asm/io.h @@ -1,6 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _M68K_IO_H +#define _M68K_IO_H + #if defined(__uClinux__) || defined(CONFIG_COLDFIRE) #include <asm/io_no.h> #else #include <asm/io_mm.h> #endif + +#include <asm-generic/io.h> + +#endif /* _M68K_IO_H */ diff --git a/arch/m68k/include/asm/io_mm.h b/arch/m68k/include/asm/io_mm.h index fe485f4f5fac..782b78f8a048 100644 --- a/arch/m68k/include/asm/io_mm.h +++ b/arch/m68k/include/asm/io_mm.h @@ -16,13 +16,11 @@ * isa_readX(),isa_writeX() are for ISA memory */ -#ifndef _IO_H -#define _IO_H +#ifndef _M68K_IO_MM_H +#define _M68K_IO_MM_H #ifdef __KERNEL__ -#define ARCH_HAS_IOREMAP_WT - #include <linux/compiler.h> #include <asm/raw_io.h> #include <asm/virtconvert.h> @@ -369,40 +367,6 @@ static inline void isa_delay(void) #define writew(val, addr) out_le16((addr), (val)) #endif /* CONFIG_ATARI_ROM_ISA */ -#if !defined(CONFIG_ISA) && !defined(CONFIG_ATARI_ROM_ISA) -/* - * We need to define dummy functions for GENERIC_IOMAP support. - */ -#define inb(port) 0xff -#define inb_p(port) 0xff -#define outb(val,port) ((void)0) -#define outb_p(val,port) ((void)0) -#define inw(port) 0xffff -#define inw_p(port) 0xffff -#define outw(val,port) ((void)0) -#define outw_p(val,port) ((void)0) -#define inl(port) 0xffffffffUL -#define inl_p(port) 0xffffffffUL -#define outl(val,port) ((void)0) -#define outl_p(val,port) ((void)0) - -#define insb(port,buf,nr) ((void)0) -#define outsb(port,buf,nr) ((void)0) -#define insw(port,buf,nr) ((void)0) -#define outsw(port,buf,nr) ((void)0) -#define insl(port,buf,nr) ((void)0) -#define outsl(port,buf,nr) ((void)0) - -/* - * These should be valid on any ioremap()ed region - */ -#define readb(addr) in_8(addr) -#define writeb(val,addr) out_8((addr),(val)) -#define readw(addr) in_le16(addr) -#define writew(val,addr) out_le16((addr),(val)) - -#endif /* !CONFIG_ISA && !CONFIG_ATARI_ROM_ISA */ - #define readl(addr) in_le32(addr) #define writel(val,addr) out_le32((addr),(val)) @@ -444,4 +408,4 @@ static inline void isa_delay(void) #define writew_relaxed(b, addr) writew(b, addr) #define writel_relaxed(b, addr) writel(b, addr) -#endif /* _IO_H */ +#endif /* _M68K_IO_MM_H */ diff --git a/arch/m68k/include/asm/io_no.h b/arch/m68k/include/asm/io_no.h index 83a0a6d449f4..0498192e1d98 100644 --- a/arch/m68k/include/asm/io_no.h +++ b/arch/m68k/include/asm/io_no.h @@ -131,19 +131,7 @@ static inline void writel(u32 value, volatile void __iomem *addr) #define PCI_SPACE_LIMIT PCI_IO_MASK #endif /* CONFIG_PCI */ -/* - * These are defined in kmap.h as static inline functions. To maintain - * previous behavior we put these define guards here so io_mm.h doesn't - * see them. - */ -#ifdef CONFIG_MMU -#define memset_io memset_io -#define memcpy_fromio memcpy_fromio -#define memcpy_toio memcpy_toio -#endif - #include <asm/kmap.h> #include <asm/virtconvert.h> -#include <asm-generic/io.h> #endif /* _M68KNOMMU_IO_H */ diff --git a/arch/m68k/include/asm/kmap.h b/arch/m68k/include/asm/kmap.h index 84b8333db8ad..aac7f045f7f0 100644 --- a/arch/m68k/include/asm/kmap.h +++ b/arch/m68k/include/asm/kmap.h @@ -4,6 +4,8 @@ #ifdef CONFIG_MMU +#define ARCH_HAS_IOREMAP_WT + /* Values for nocacheflag and cmode */ #define IOMAP_FULL_CACHING 0 #define IOMAP_NOCACHE_SER 1 @@ -16,6 +18,7 @@ */ extern void __iomem *__ioremap(unsigned long physaddr, unsigned long size, int cacheflag); +#define iounmap iounmap extern void iounmap(void __iomem *addr); extern void __iounmap(void *addr, unsigned long size); @@ -33,31 +36,35 @@ static inline void __iomem *ioremap_nocache(unsigned long physaddr, } #define ioremap_uc ioremap_nocache +#define ioremap_wt ioremap_wt static inline void __iomem *ioremap_wt(unsigned long physaddr, unsigned long size) { return __ioremap(physaddr, size, IOMAP_WRITETHROUGH); } -#define ioremap_fillcache ioremap_fullcache +#define ioremap_fullcache ioremap_fullcache static inline void __iomem *ioremap_fullcache(unsigned long physaddr, unsigned long size) { return __ioremap(physaddr, size, IOMAP_FULL_CACHING); } +#define memset_io memset_io static inline void memset_io(volatile void __iomem *addr, unsigned char val, int count) { __builtin_memset((void __force *) addr, val, count); } +#define memcpy_fromio memcpy_fromio static inline void memcpy_fromio(void *dst, const volatile void __iomem *src, int count) { __builtin_memcpy(dst, (void __force *) src, count); } +#define memcpy_toio memcpy_toio static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int count) { diff --git a/arch/m68k/include/asm/machdep.h b/arch/m68k/include/asm/machdep.h index 1605da48ebf2..49bd3266b4b1 100644 --- a/arch/m68k/include/asm/machdep.h +++ b/arch/m68k/include/asm/machdep.h @@ -22,7 +22,6 @@ extern int (*mach_hwclk)(int, struct rtc_time*); extern unsigned int (*mach_get_ss)(void); extern int (*mach_get_rtc_pll)(struct rtc_pll_info *); extern int (*mach_set_rtc_pll)(struct rtc_pll_info *); -extern int (*mach_set_clock_mmss)(unsigned long); extern void (*mach_reset)( void ); extern void (*mach_halt)( void ); extern void (*mach_power_off)( void ); diff --git a/arch/m68k/include/asm/macintosh.h b/arch/m68k/include/asm/macintosh.h index 9b840c03ebb7..08cee11180e6 100644 --- a/arch/m68k/include/asm/macintosh.h +++ b/arch/m68k/include/asm/macintosh.h @@ -57,7 +57,6 @@ struct mac_model #define MAC_SCSI_IIFX 5 #define MAC_SCSI_DUO 6 #define MAC_SCSI_LC 7 -#define MAC_SCSI_LATE 8 #define MAC_IDE_NONE 0 #define MAC_IDE_QUADRA 1 diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h index 8b707c249026..12fe700632f4 100644 --- a/arch/m68k/include/asm/mcf_pgalloc.h +++ b/arch/m68k/include/asm/mcf_pgalloc.h @@ -44,6 +44,7 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, unsigned long address) { + pgtable_page_dtor(page); __free_page(page); } @@ -74,8 +75,9 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, return page; } -extern inline void pte_free(struct mm_struct *mm, struct page *page) +static inline void pte_free(struct mm_struct *mm, struct page *page) { + pgtable_page_dtor(page); __free_page(page); } diff --git a/arch/m68k/include/asm/page_no.h b/arch/m68k/include/asm/page_no.h index e644c4daf540..6bbe52025de3 100644 --- a/arch/m68k/include/asm/page_no.h +++ b/arch/m68k/include/asm/page_no.h @@ -18,7 +18,7 @@ extern unsigned long memory_end; #define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE #define __pa(vaddr) ((unsigned long)(vaddr)) -#define __va(paddr) ((void *)(paddr)) +#define __va(paddr) ((void *)((unsigned long)(paddr))) #define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT) #define pfn_to_virt(pfn) __va((pfn) << PAGE_SHIFT) diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c index 463572c4943f..e99993c57d6b 100644 --- a/arch/m68k/kernel/dma.c +++ b/arch/m68k/kernel/dma.c @@ -6,7 +6,7 @@ #undef DEBUG -#include <linux/dma-mapping.h> +#include <linux/dma-noncoherent.h> #include <linux/device.h> #include <linux/kernel.h> #include <linux/platform_device.h> @@ -19,7 +19,7 @@ #if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE) -static void *m68k_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, +void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t flag, unsigned long attrs) { struct page *page, **map; @@ -62,7 +62,7 @@ static void *m68k_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, return addr; } -static void m68k_dma_free(struct device *dev, size_t size, void *addr, +void arch_dma_free(struct device *dev, size_t size, void *addr, dma_addr_t handle, unsigned long attrs) { pr_debug("dma_free_coherent: %p, %x\n", addr, handle); @@ -73,8 +73,8 @@ static void m68k_dma_free(struct device *dev, size_t size, void *addr, #include <asm/cacheflush.h> -static void *m68k_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs) { void *ret; @@ -89,7 +89,7 @@ static void *m68k_dma_alloc(struct device *dev, size_t size, return ret; } -static void m68k_dma_free(struct device *dev, size_t size, void *vaddr, +void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { free_pages((unsigned long)vaddr, get_order(size)); @@ -97,8 +97,8 @@ static void m68k_dma_free(struct device *dev, size_t size, void *vaddr, #endif /* CONFIG_MMU && !CONFIG_COLDFIRE */ -static void m68k_dma_sync_single_for_device(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(struct device *dev, phys_addr_t handle, + size_t size, enum dma_data_direction dir) { switch (dir) { case DMA_BIDIRECTIONAL: @@ -115,58 +115,6 @@ static void m68k_dma_sync_single_for_device(struct device *dev, } } -static void m68k_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sglist, int nents, enum dma_data_direction dir) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sglist, sg, nents, i) { - dma_sync_single_for_device(dev, sg->dma_address, sg->length, - dir); - } -} - -static dma_addr_t m68k_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t handle = page_to_phys(page) + offset; - - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - dma_sync_single_for_device(dev, handle, size, dir); - - return handle; -} - -static int m68k_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sglist, sg, nents, i) { - sg->dma_address = sg_phys(sg); - - if (attrs & DMA_ATTR_SKIP_CPU_SYNC) - continue; - - dma_sync_single_for_device(dev, sg->dma_address, sg->length, - dir); - } - return nents; -} - -const struct dma_map_ops m68k_dma_ops = { - .alloc = m68k_dma_alloc, - .free = m68k_dma_free, - .map_page = m68k_dma_map_page, - .map_sg = m68k_dma_map_sg, - .sync_single_for_device = m68k_dma_sync_single_for_device, - .sync_sg_for_device = m68k_dma_sync_sg_for_device, -}; -EXPORT_SYMBOL(m68k_dma_ops); - void arch_setup_pdev_archdata(struct platform_device *pdev) { if (pdev->dev.coherent_dma_mask == DMA_MASK_NONE && diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index f35e3ebd6331..5d3596c180f9 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -21,6 +21,7 @@ #include <linux/string.h> #include <linux/init.h> #include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/module.h> @@ -88,7 +89,6 @@ void (*mach_get_hardware_list) (struct seq_file *m); /* machine dependent timer functions */ int (*mach_hwclk) (int, struct rtc_time*); EXPORT_SYMBOL(mach_hwclk); -int (*mach_set_clock_mmss) (unsigned long); unsigned int (*mach_get_ss)(void); int (*mach_get_rtc_pll)(struct rtc_pll_info *); int (*mach_set_rtc_pll)(struct rtc_pll_info *); @@ -165,6 +165,8 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record) be32_to_cpu(m->addr); m68k_memory[m68k_num_memory].size = be32_to_cpu(m->size); + memblock_add(m68k_memory[m68k_num_memory].addr, + m68k_memory[m68k_num_memory].size); m68k_num_memory++; } else pr_warn("%s: too many memory chunks\n", @@ -224,10 +226,6 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record) void __init setup_arch(char **cmdline_p) { -#ifndef CONFIG_SUN3 - int i; -#endif - /* The bootinfo is located right after the kernel */ if (!CPU_IS_COLDFIRE) m68k_parse_bootinfo((const struct bi_record *)_end); @@ -356,14 +354,9 @@ void __init setup_arch(char **cmdline_p) #endif #ifndef CONFIG_SUN3 - for (i = 1; i < m68k_num_memory; i++) - free_bootmem_node(NODE_DATA(i), m68k_memory[i].addr, - m68k_memory[i].size); #ifdef CONFIG_BLK_DEV_INITRD if (m68k_ramdisk.size) { - reserve_bootmem_node(__virt_to_node(phys_to_virt(m68k_ramdisk.addr)), - m68k_ramdisk.addr, m68k_ramdisk.size, - BOOTMEM_DEFAULT); + memblock_reserve(m68k_ramdisk.addr, m68k_ramdisk.size); initrd_start = (unsigned long)phys_to_virt(m68k_ramdisk.addr); initrd_end = initrd_start + m68k_ramdisk.size; pr_info("initrd: %08lx - %08lx\n", initrd_start, initrd_end); diff --git a/arch/m68k/kernel/setup_no.c b/arch/m68k/kernel/setup_no.c index a98af1018201..cfd5475bfc31 100644 --- a/arch/m68k/kernel/setup_no.c +++ b/arch/m68k/kernel/setup_no.c @@ -28,6 +28,7 @@ #include <linux/errno.h> #include <linux/string.h> #include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/seq_file.h> #include <linux/init.h> #include <linux/initrd.h> @@ -51,7 +52,6 @@ char __initdata command_line[COMMAND_LINE_SIZE]; /* machine dependent timer functions */ void (*mach_sched_init)(irq_handler_t handler) __initdata = NULL; -int (*mach_set_clock_mmss)(unsigned long); int (*mach_hwclk) (int, struct rtc_time*); /* machine dependent reboot functions */ @@ -86,8 +86,6 @@ void (*mach_power_off)(void); void __init setup_arch(char **cmdline_p) { - int bootmap_size; - memory_start = PAGE_ALIGN(_ramstart); memory_end = _ramend; @@ -142,6 +140,8 @@ void __init setup_arch(char **cmdline_p) pr_debug("MEMORY -> ROMFS=0x%p-0x%06lx MEM=0x%06lx-0x%06lx\n ", __bss_stop, memory_start, memory_start, memory_end); + memblock_add(memory_start, memory_end - memory_start); + /* Keep a copy of command line */ *cmdline_p = &command_line[0]; memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE); @@ -158,23 +158,10 @@ void __init setup_arch(char **cmdline_p) min_low_pfn = PFN_DOWN(memory_start); max_pfn = max_low_pfn = PFN_DOWN(memory_end); - bootmap_size = init_bootmem_node( - NODE_DATA(0), - min_low_pfn, /* map goes here */ - PFN_DOWN(PAGE_OFFSET), - max_pfn); - /* - * Free the usable memory, we have to make sure we do not free - * the bootmem bitmap so we then reserve it after freeing it :-) - */ - free_bootmem(memory_start, memory_end - memory_start); - reserve_bootmem(memory_start, bootmap_size, BOOTMEM_DEFAULT); - #if defined(CONFIG_UBOOT) && defined(CONFIG_BLK_DEV_INITRD) if ((initrd_start > 0) && (initrd_start < initrd_end) && (initrd_end < memory_end)) - reserve_bootmem(initrd_start, initrd_end - initrd_start, - BOOTMEM_DEFAULT); + memblock_reserve(initrd_start, initrd_end - initrd_start); #endif /* if defined(CONFIG_BLK_DEV_INITRD) */ /* diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index e522307db47c..b02d7254b73a 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -57,7 +57,6 @@ static unsigned long mac_orig_videoaddr; /* Mac specific timer functions */ extern u32 mac_gettimeoffset(void); extern int mac_hwclk(int, struct rtc_time *); -extern int mac_set_clock_mmss(unsigned long); extern void iop_preinit(void); extern void iop_init(void); extern void via_init(void); @@ -158,7 +157,6 @@ void __init config_mac(void) mach_get_model = mac_get_model; arch_gettimeoffset = mac_gettimeoffset; mach_hwclk = mac_hwclk; - mach_set_clock_mmss = mac_set_clock_mmss; mach_reset = mac_reset; mach_halt = mac_poweroff; mach_power_off = mac_poweroff; @@ -709,7 +707,7 @@ static struct mac_model mac_data_table[] = { .name = "PowerBook 520", .adb_type = MAC_ADB_PB2, .via_type = MAC_VIA_QUADRA, - .scsi_type = MAC_SCSI_LATE, + .scsi_type = MAC_SCSI_OLD, .scc_type = MAC_SCC_QUADRA, .ether_type = MAC_ETHER_SONIC, .floppy_type = MAC_FLOPPY_SWIM_ADDR2, @@ -943,18 +941,6 @@ static const struct resource mac_scsi_old_rsrc[] __initconst = { }, }; -static const struct resource mac_scsi_late_rsrc[] __initconst = { - { - .flags = IORESOURCE_IRQ, - .start = IRQ_MAC_SCSI, - .end = IRQ_MAC_SCSI, - }, { - .flags = IORESOURCE_MEM, - .start = 0x50010000, - .end = 0x50011FFF, - }, -}; - static const struct resource mac_scsi_ccl_rsrc[] __initconst = { { .flags = IORESOURCE_IRQ, @@ -1064,11 +1050,6 @@ int __init mac_platform_init(void) platform_device_register_simple("mac_scsi", 0, mac_scsi_old_rsrc, ARRAY_SIZE(mac_scsi_old_rsrc)); break; - case MAC_SCSI_LATE: - /* XXX PDMA support for PowerBook 500 series needs testing */ - platform_device_register_simple("mac_scsi", 0, - mac_scsi_late_rsrc, ARRAY_SIZE(mac_scsi_late_rsrc)); - break; case MAC_SCSI_LC: /* Addresses from Mac LC data in Designing Cards & Drivers 3ed. * Also from the Developer Notes for Classic II, LC III, diff --git a/arch/m68k/mac/misc.c b/arch/m68k/mac/misc.c index c68054361615..19e9d8eef1f2 100644 --- a/arch/m68k/mac/misc.c +++ b/arch/m68k/mac/misc.c @@ -26,33 +26,38 @@ #include <asm/machdep.h> -/* Offset between Unix time (1970-based) and Mac time (1904-based) */ +/* + * Offset between Unix time (1970-based) and Mac time (1904-based). Cuda and PMU + * times wrap in 2040. If we need to handle later times, the read_time functions + * need to be changed to interpret wrapped times as post-2040. + */ #define RTC_OFFSET 2082844800 static void (*rom_reset)(void); #ifdef CONFIG_ADB_CUDA -static long cuda_read_time(void) +static time64_t cuda_read_time(void) { struct adb_request req; - long time; + time64_t time; if (cuda_request(&req, NULL, 2, CUDA_PACKET, CUDA_GET_TIME) < 0) return 0; while (!req.complete) cuda_poll(); - time = (req.reply[3] << 24) | (req.reply[4] << 16) | - (req.reply[5] << 8) | req.reply[6]; + time = (u32)((req.reply[3] << 24) | (req.reply[4] << 16) | + (req.reply[5] << 8) | req.reply[6]); + return time - RTC_OFFSET; } -static void cuda_write_time(long data) +static void cuda_write_time(time64_t time) { struct adb_request req; + u32 data = lower_32_bits(time + RTC_OFFSET); - data += RTC_OFFSET; if (cuda_request(&req, NULL, 6, CUDA_PACKET, CUDA_SET_TIME, (data >> 24) & 0xFF, (data >> 16) & 0xFF, (data >> 8) & 0xFF, data & 0xFF) < 0) @@ -86,26 +91,27 @@ static void cuda_write_pram(int offset, __u8 data) #endif /* CONFIG_ADB_CUDA */ #ifdef CONFIG_ADB_PMU68K -static long pmu_read_time(void) +static time64_t pmu_read_time(void) { struct adb_request req; - long time; + time64_t time; if (pmu_request(&req, NULL, 1, PMU_READ_RTC) < 0) return 0; while (!req.complete) pmu_poll(); - time = (req.reply[1] << 24) | (req.reply[2] << 16) | - (req.reply[3] << 8) | req.reply[4]; + time = (u32)((req.reply[1] << 24) | (req.reply[2] << 16) | + (req.reply[3] << 8) | req.reply[4]); + return time - RTC_OFFSET; } -static void pmu_write_time(long data) +static void pmu_write_time(time64_t time) { struct adb_request req; + u32 data = lower_32_bits(time + RTC_OFFSET); - data += RTC_OFFSET; if (pmu_request(&req, NULL, 5, PMU_SET_RTC, (data >> 24) & 0xFF, (data >> 16) & 0xFF, (data >> 8) & 0xFF, data & 0xFF) < 0) @@ -245,11 +251,11 @@ static void via_write_pram(int offset, __u8 data) * is basically any machine with Mac II-style ADB. */ -static long via_read_time(void) +static time64_t via_read_time(void) { union { __u8 cdata[4]; - long idata; + __u32 idata; } result, last_result; int count = 1; @@ -270,7 +276,7 @@ static long via_read_time(void) via_pram_command(0x8D, &result.cdata[0]); if (result.idata == last_result.idata) - return result.idata - RTC_OFFSET; + return (time64_t)result.idata - RTC_OFFSET; if (++count > 10) break; @@ -278,8 +284,8 @@ static long via_read_time(void) last_result.idata = result.idata; } - pr_err("via_read_time: failed to read a stable value; got 0x%08lx then 0x%08lx\n", - last_result.idata, result.idata); + pr_err("%s: failed to read a stable value; got 0x%08x then 0x%08x\n", + __func__, last_result.idata, result.idata); return 0; } @@ -291,11 +297,11 @@ static long via_read_time(void) * is basically any machine with Mac II-style ADB. */ -static void via_write_time(long time) +static void via_write_time(time64_t time) { union { __u8 cdata[4]; - long idata; + __u32 idata; } data; __u8 temp; @@ -304,7 +310,7 @@ static void via_write_time(long time) temp = 0x55; via_pram_command(0x35, &temp); - data.idata = time + RTC_OFFSET; + data.idata = lower_32_bits(time + RTC_OFFSET); via_pram_command(0x01, &data.cdata[3]); via_pram_command(0x05, &data.cdata[2]); via_pram_command(0x09, &data.cdata[1]); @@ -585,12 +591,15 @@ void mac_reset(void) * This function translates seconds since 1970 into a proper date. * * Algorithm cribbed from glibc2.1, __offtime(). + * + * This is roughly same as rtc_time64_to_tm(), which we should probably + * use here, but it's only available when CONFIG_RTC_LIB is enabled. */ #define SECS_PER_MINUTE (60) #define SECS_PER_HOUR (SECS_PER_MINUTE * 60) #define SECS_PER_DAY (SECS_PER_HOUR * 24) -static void unmktime(unsigned long time, long offset, +static void unmktime(time64_t time, long offset, int *yearp, int *monp, int *dayp, int *hourp, int *minp, int *secp) { @@ -602,11 +611,10 @@ static void unmktime(unsigned long time, long offset, /* Leap years. */ { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 } }; - long int days, rem, y, wday, yday; + int days, rem, y, wday, yday; const unsigned short int *ip; - days = time / SECS_PER_DAY; - rem = time % SECS_PER_DAY; + days = div_u64_rem(time, SECS_PER_DAY, &rem); rem += offset; while (rem < 0) { rem += SECS_PER_DAY; @@ -657,7 +665,7 @@ static void unmktime(unsigned long time, long offset, int mac_hwclk(int op, struct rtc_time *t) { - unsigned long now; + time64_t now; if (!op) { /* read */ switch (macintosh_config->adb_type) { @@ -693,8 +701,8 @@ int mac_hwclk(int op, struct rtc_time *t) __func__, t->tm_year + 1900, t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec); - now = mktime(t->tm_year + 1900, t->tm_mon + 1, t->tm_mday, - t->tm_hour, t->tm_min, t->tm_sec); + now = mktime64(t->tm_year + 1900, t->tm_mon + 1, t->tm_mday, + t->tm_hour, t->tm_min, t->tm_sec); switch (macintosh_config->adb_type) { case MAC_ADB_IOP: @@ -719,19 +727,3 @@ int mac_hwclk(int op, struct rtc_time *t) } return 0; } - -/* - * Set minutes/seconds in the hardware clock - */ - -int mac_set_clock_mmss (unsigned long nowtime) -{ - struct rtc_time now; - - mac_hwclk(0, &now); - now.tm_sec = nowtime % 60; - now.tm_min = (nowtime / 60) % 60; - mac_hwclk(1, &now); - - return 0; -} diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c index 8827b7f91402..38e2b272c220 100644 --- a/arch/m68k/mm/init.c +++ b/arch/m68k/mm/init.c @@ -71,7 +71,6 @@ void __init m68k_setup_node(int node) pg_data_table[i] = pg_data_map + node; } #endif - pg_data_map[node].bdata = bootmem_node_data + node; node_set_online(node); } diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c index 2925d795d71a..70dde040779b 100644 --- a/arch/m68k/mm/mcfmmu.c +++ b/arch/m68k/mm/mcfmmu.c @@ -14,6 +14,7 @@ #include <linux/init.h> #include <linux/string.h> #include <linux/bootmem.h> +#include <linux/memblock.h> #include <asm/setup.h> #include <asm/page.h> @@ -153,31 +154,31 @@ int cf_tlb_miss(struct pt_regs *regs, int write, int dtlb, int extension_word) void __init cf_bootmem_alloc(void) { - unsigned long start_pfn; unsigned long memstart; /* _rambase and _ramend will be naturally page aligned */ m68k_memory[0].addr = _rambase; m68k_memory[0].size = _ramend - _rambase; + memblock_add(m68k_memory[0].addr, m68k_memory[0].size); + /* compute total pages in system */ num_pages = PFN_DOWN(_ramend - _rambase); /* page numbers */ memstart = PAGE_ALIGN(_ramstart); min_low_pfn = PFN_DOWN(_rambase); - start_pfn = PFN_DOWN(memstart); max_pfn = max_low_pfn = PFN_DOWN(_ramend); high_memory = (void *)_ramend; + /* Reserve kernel text/data/bss */ + memblock_reserve(memstart, memstart - _rambase); + m68k_virt_to_node_shift = fls(_ramend - 1) - 6; module_fixup(NULL, __start_fixup, __stop_fixup); - /* setup bootmem data */ + /* setup node data */ m68k_setup_node(0); - memstart += init_bootmem_node(NODE_DATA(0), start_pfn, - min_low_pfn, max_low_pfn); - free_bootmem_node(NODE_DATA(0), memstart, _ramend - memstart); } /* diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index e490ecc7842c..4e17ecb5928a 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -19,6 +19,7 @@ #include <linux/types.h> #include <linux/init.h> #include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/gfp.h> #include <asm/setup.h> @@ -208,7 +209,7 @@ void __init paging_init(void) { unsigned long zones_size[MAX_NR_ZONES] = { 0, }; unsigned long min_addr, max_addr; - unsigned long addr, size, end; + unsigned long addr; int i; #ifdef DEBUG @@ -253,34 +254,20 @@ void __init paging_init(void) min_low_pfn = availmem >> PAGE_SHIFT; max_pfn = max_low_pfn = max_addr >> PAGE_SHIFT; - for (i = 0; i < m68k_num_memory; i++) { - addr = m68k_memory[i].addr; - end = addr + m68k_memory[i].size; - m68k_setup_node(i); - availmem = PAGE_ALIGN(availmem); - availmem += init_bootmem_node(NODE_DATA(i), - availmem >> PAGE_SHIFT, - addr >> PAGE_SHIFT, - end >> PAGE_SHIFT); - } + /* Reserve kernel text/data/bss and the memory allocated in head.S */ + memblock_reserve(m68k_memory[0].addr, availmem - m68k_memory[0].addr); /* * Map the physical memory available into the kernel virtual - * address space. First initialize the bootmem allocator with - * the memory we already mapped, so map_node() has something - * to allocate. + * address space. Make sure memblock will not try to allocate + * pages beyond the memory we already mapped in head.S */ - addr = m68k_memory[0].addr; - size = m68k_memory[0].size; - free_bootmem_node(NODE_DATA(0), availmem, - min(m68k_init_mapped_size, size) - (availmem - addr)); - map_node(0); - if (size > m68k_init_mapped_size) - free_bootmem_node(NODE_DATA(0), addr + m68k_init_mapped_size, - size - m68k_init_mapped_size); - - for (i = 1; i < m68k_num_memory; i++) + memblock_set_bottom_up(true); + + for (i = 0; i < m68k_num_memory; i++) { + m68k_setup_node(i); map_node(i); + } flush_tlb_all(); diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c index f8a710fd84cd..adea549d240e 100644 --- a/arch/m68k/mvme147/config.c +++ b/arch/m68k/mvme147/config.c @@ -40,7 +40,6 @@ static void mvme147_get_model(char *model); extern void mvme147_sched_init(irq_handler_t handler); extern u32 mvme147_gettimeoffset(void); extern int mvme147_hwclk (int, struct rtc_time *); -extern int mvme147_set_clock_mmss (unsigned long); extern void mvme147_reset (void); @@ -92,7 +91,6 @@ void __init config_mvme147(void) mach_init_IRQ = mvme147_init_IRQ; arch_gettimeoffset = mvme147_gettimeoffset; mach_hwclk = mvme147_hwclk; - mach_set_clock_mmss = mvme147_set_clock_mmss; mach_reset = mvme147_reset; mach_get_model = mvme147_get_model; @@ -164,8 +162,3 @@ int mvme147_hwclk(int op, struct rtc_time *t) } return 0; } - -int mvme147_set_clock_mmss (unsigned long nowtime) -{ - return 0; -} diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c index 4ffd9ef98de4..6ee36a5b528d 100644 --- a/arch/m68k/mvme16x/config.c +++ b/arch/m68k/mvme16x/config.c @@ -46,7 +46,6 @@ static void mvme16x_get_model(char *model); extern void mvme16x_sched_init(irq_handler_t handler); extern u32 mvme16x_gettimeoffset(void); extern int mvme16x_hwclk (int, struct rtc_time *); -extern int mvme16x_set_clock_mmss (unsigned long); extern void mvme16x_reset (void); int bcd2int (unsigned char b); @@ -280,7 +279,6 @@ void __init config_mvme16x(void) mach_init_IRQ = mvme16x_init_IRQ; arch_gettimeoffset = mvme16x_gettimeoffset; mach_hwclk = mvme16x_hwclk; - mach_set_clock_mmss = mvme16x_set_clock_mmss; mach_reset = mvme16x_reset; mach_get_model = mvme16x_get_model; mach_get_hardware_list = mvme16x_get_hardware_list; @@ -411,9 +409,3 @@ int mvme16x_hwclk(int op, struct rtc_time *t) } return 0; } - -int mvme16x_set_clock_mmss (unsigned long nowtime) -{ - return 0; -} - diff --git a/arch/m68k/q40/config.c b/arch/m68k/q40/config.c index 71c0867ecf20..96810d91da2b 100644 --- a/arch/m68k/q40/config.c +++ b/arch/m68k/q40/config.c @@ -43,7 +43,6 @@ extern void q40_sched_init(irq_handler_t handler); static u32 q40_gettimeoffset(void); static int q40_hwclk(int, struct rtc_time *); static unsigned int q40_get_ss(void); -static int q40_set_clock_mmss(unsigned long); static int q40_get_rtc_pll(struct rtc_pll_info *pll); static int q40_set_rtc_pll(struct rtc_pll_info *pll); @@ -175,7 +174,6 @@ void __init config_q40(void) mach_get_ss = q40_get_ss; mach_get_rtc_pll = q40_get_rtc_pll; mach_set_rtc_pll = q40_set_rtc_pll; - mach_set_clock_mmss = q40_set_clock_mmss; mach_reset = q40_reset; mach_get_model = q40_get_model; @@ -267,34 +265,6 @@ static unsigned int q40_get_ss(void) return bcd2bin(Q40_RTC_SECS); } -/* - * Set the minutes and seconds from seconds value 'nowtime'. Fail if - * clock is out by > 30 minutes. Logic lifted from atari code. - */ - -static int q40_set_clock_mmss(unsigned long nowtime) -{ - int retval = 0; - short real_seconds = nowtime % 60, real_minutes = (nowtime / 60) % 60; - - int rtc_minutes; - - rtc_minutes = bcd2bin(Q40_RTC_MINS); - - if ((rtc_minutes < real_minutes ? - real_minutes - rtc_minutes : - rtc_minutes - real_minutes) < 30) { - Q40_RTC_CTRL |= Q40_RTC_WRITE; - Q40_RTC_MINS = bin2bcd(real_minutes); - Q40_RTC_SECS = bin2bcd(real_seconds); - Q40_RTC_CTRL &= ~(Q40_RTC_WRITE); - } else - retval = -1; - - return retval; -} - - /* get and set PLL calibration of RTC clock */ #define Q40_RTC_PLL_MASK ((1<<5)-1) #define Q40_RTC_PLL_SIGN (1<<5) diff --git a/arch/m68k/sun3/config.c b/arch/m68k/sun3/config.c index 1d28d380e8cc..79a2bb857906 100644 --- a/arch/m68k/sun3/config.c +++ b/arch/m68k/sun3/config.c @@ -123,10 +123,6 @@ static void __init sun3_bootmem_alloc(unsigned long memory_start, availmem = memory_start; m68k_setup_node(0); - availmem += init_bootmem(start_page, num_pages); - availmem = (availmem + (PAGE_SIZE-1)) & PAGE_MASK; - - free_bootmem(__pa(availmem), memory_end - (availmem)); } diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 08c10c518f83..642a56e2a1ea 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -16,6 +16,7 @@ config MIPS select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select CPU_PM if CPU_IDLE + select DMA_DIRECT_OPS select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS select GENERIC_CMOS_UPDATE @@ -97,6 +98,7 @@ config MIPS_GENERIC select HW_HAS_PCI select IRQ_MIPS_CPU select LIBFDT + select MIPS_AUTO_PFN_OFFSET select MIPS_CPU_SCACHE select MIPS_GIC select MIPS_L1_CACHE_SHIFT_7 @@ -193,6 +195,7 @@ config ATH79 select CSRC_R4K select DMA_NONCOHERENT select GPIOLIB + select PINCTRL select HAVE_CLK select COMMON_CLK select CLKDEV_LOOKUP @@ -211,6 +214,8 @@ config ATH79 config BMIPS_GENERIC bool "Broadcom Generic BMIPS kernel" + select ARCH_HAS_SYNC_DMA_FOR_CPU_ALL + select ARCH_HAS_PHYS_TO_DMA select BOOT_RAW select NO_EXCEPT_FILL select USE_OF @@ -438,7 +443,6 @@ config MACH_LOONGSON32 config MACH_LOONGSON64 bool "Loongson-2/3 family of machines" - select ARCH_HAS_PHYS_TO_DMA select SYS_SUPPORTS_ZBOOT help This enables the support of Loongson-2/3 family of machines. @@ -662,11 +666,11 @@ config SGI_IP22 config SGI_IP27 bool "SGI IP27 (Origin200/2000)" + select ARCH_HAS_PHYS_TO_DMA select FW_ARC select FW_ARC64 select BOOT_ELF64 select DEFAULT_SGI_PARTITION - select DMA_COHERENT select SYS_HAS_EARLY_PRINTK select HW_HAS_PCI select NR_CPUS_DEFAULT_64 @@ -721,6 +725,7 @@ config SGI_IP28 config SGI_IP32 bool "SGI IP32 (O2)" + select ARCH_HAS_PHYS_TO_DMA select FW_ARC select FW_ARC32 select BOOT_ELF32 @@ -743,7 +748,6 @@ config SGI_IP32 config SIBYTE_CRHINE bool "Sibyte BCM91120C-CRhine" select BOOT_ELF32 - select DMA_COHERENT select SIBYTE_BCM1120 select SWAP_IO_SPACE select SYS_HAS_CPU_SB1 @@ -753,7 +757,6 @@ config SIBYTE_CRHINE config SIBYTE_CARMEL bool "Sibyte BCM91120x-Carmel" select BOOT_ELF32 - select DMA_COHERENT select SIBYTE_BCM1120 select SWAP_IO_SPACE select SYS_HAS_CPU_SB1 @@ -763,7 +766,6 @@ config SIBYTE_CARMEL config SIBYTE_CRHONE bool "Sibyte BCM91125C-CRhone" select BOOT_ELF32 - select DMA_COHERENT select SIBYTE_BCM1125 select SWAP_IO_SPACE select SYS_HAS_CPU_SB1 @@ -774,7 +776,6 @@ config SIBYTE_CRHONE config SIBYTE_RHONE bool "Sibyte BCM91125E-Rhone" select BOOT_ELF32 - select DMA_COHERENT select SIBYTE_BCM1125H select SWAP_IO_SPACE select SYS_HAS_CPU_SB1 @@ -784,7 +785,6 @@ config SIBYTE_RHONE config SIBYTE_SWARM bool "Sibyte BCM91250A-SWARM" select BOOT_ELF32 - select DMA_COHERENT select HAVE_PATA_PLATFORM select SIBYTE_SB1250 select SWAP_IO_SPACE @@ -797,7 +797,6 @@ config SIBYTE_SWARM config SIBYTE_LITTLESUR bool "Sibyte BCM91250C2-LittleSur" select BOOT_ELF32 - select DMA_COHERENT select HAVE_PATA_PLATFORM select SIBYTE_SB1250 select SWAP_IO_SPACE @@ -809,7 +808,6 @@ config SIBYTE_LITTLESUR config SIBYTE_SENTOSA bool "Sibyte BCM91250E-Sentosa" select BOOT_ELF32 - select DMA_COHERENT select SIBYTE_SB1250 select SWAP_IO_SPACE select SYS_HAS_CPU_SB1 @@ -819,7 +817,6 @@ config SIBYTE_SENTOSA config SIBYTE_BIGSUR bool "Sibyte BCM91480B-BigSur" select BOOT_ELF32 - select DMA_COHERENT select NR_CPUS_DEFAULT_4 select SIBYTE_BCM1x80 select SWAP_IO_SPACE @@ -895,8 +892,8 @@ config CAVIUM_OCTEON_SOC bool "Cavium Networks Octeon SoC based boards" select CEVT_R4K select ARCH_HAS_PHYS_TO_DMA + select HAS_RAPIDIO select PHYS_ADDR_T_64BIT - select DMA_COHERENT select SYS_SUPPORTS_64BIT_KERNEL select SYS_SUPPORTS_BIG_ENDIAN select EDAC_SUPPORT @@ -945,7 +942,6 @@ config NLM_XLR_BOARD select PHYS_ADDR_T_64BIT select SYS_SUPPORTS_BIG_ENDIAN select SYS_SUPPORTS_HIGHMEM - select DMA_COHERENT select NR_CPUS_DEFAULT_32 select CEVT_R4K select CSRC_R4K @@ -973,7 +969,6 @@ config NLM_XLP_BOARD select SYS_SUPPORTS_BIG_ENDIAN select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_HIGHMEM - select DMA_COHERENT select NR_CPUS_DEFAULT_32 select CEVT_R4K select CSRC_R4K @@ -992,7 +987,6 @@ config MIPS_PARAVIRT bool "Para-Virtualized guest system" select CEVT_R4K select CSRC_R4K - select DMA_COHERENT select SYS_SUPPORTS_64BIT_KERNEL select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_BIG_ENDIAN @@ -1118,12 +1112,14 @@ config DMA_PERDEV_COHERENT bool select DMA_MAYBE_COHERENT -config DMA_COHERENT - bool - config DMA_NONCOHERENT bool + select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select ARCH_HAS_SYNC_DMA_FOR_CPU select NEED_DMA_MAP_STATE + select DMA_NONCOHERENT_MMAP + select DMA_NONCOHERENT_CACHE_SYNC + select DMA_NONCOHERENT_OPS config SYS_HAS_EARLY_PRINTK bool @@ -1365,6 +1361,7 @@ choice config CPU_LOONGSON3 bool "Loongson 3 CPU" depends on SYS_HAS_CPU_LOONGSON3 + select ARCH_HAS_PHYS_TO_DMA select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_HUGEPAGES @@ -1427,7 +1424,8 @@ config CPU_LOONGSON1B select LEDS_GPIO_REGISTER help The Loongson 1B is a 32-bit SoC, which implements the MIPS32 - release 2 instruction set. + Release 1 instruction set and part of the MIPS32 Release 2 + instruction set. config CPU_LOONGSON1C bool "Loongson 1C" @@ -1436,7 +1434,8 @@ config CPU_LOONGSON1C select LEDS_GPIO_REGISTER help The Loongson 1C is a 32-bit SoC, which implements the MIPS32 - release 2 instruction set. + Release 1 instruction set and part of the MIPS32 Release 2 + instruction set. config CPU_MIPS32_R1 bool "MIPS32 Release 1" @@ -1831,11 +1830,12 @@ config CPU_LOONGSON2 select CPU_SUPPORTS_64BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_HUGEPAGES + select ARCH_HAS_PHYS_TO_DMA config CPU_LOONGSON1 bool select CPU_MIPS32 - select CPU_MIPSR2 + select CPU_MIPSR1 select CPU_HAS_PREFETCH select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_HIGHMEM @@ -1979,12 +1979,6 @@ config SYS_HAS_CPU_XLR config SYS_HAS_CPU_XLP bool -config MIPS_MALTA_PM - depends on MIPS_MALTA - depends on PCI - bool - default y - # # CPU may reorder R->R, R->W, W->R, W->W # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC @@ -2994,6 +2988,9 @@ config PGTABLE_LEVELS default 3 if 64BIT && !PAGE_SIZE_64KB default 2 +config MIPS_AUTO_PFN_OFFSET + bool + source "init/Kconfig" source "kernel/Kconfig.freezer" @@ -3115,10 +3112,13 @@ config ZONE_DMA32 source "drivers/pcmcia/Kconfig" +config HAS_RAPIDIO + bool + default n + config RAPIDIO tristate "RapidIO support" - depends on PCI - default n + depends on HAS_RAPIDIO || PCI help If you say Y here, the kernel will include drivers and infrastructure code to support RapidIO interconnect devices. diff --git a/arch/mips/Makefile b/arch/mips/Makefile index e2122cca4ae2..5425df002a6b 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -122,12 +122,22 @@ cflags-y += -ffreestanding # are used, so we kludge that here. A bug has been filed at # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=29413. # +# clang doesn't suffer from these issues and our checks against -dumpmachine +# don't work so well when cross compiling, since without providing --target +# clang's output will be based upon the build machine. So for clang we simply +# unconditionally specify -EB or -EL as appropriate. +# +ifeq ($(cc-name),clang) +cflags-$(CONFIG_CPU_BIG_ENDIAN) += -EB +cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += -EL +else undef-all += -UMIPSEB -U_MIPSEB -U__MIPSEB -U__MIPSEB__ undef-all += -UMIPSEL -U_MIPSEL -U__MIPSEL -U__MIPSEL__ predef-be += -DMIPSEB -D_MIPSEB -D__MIPSEB -D__MIPSEB__ predef-le += -DMIPSEL -D_MIPSEL -D__MIPSEL -D__MIPSEL__ cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(shell $(CC) -dumpmachine |grep -q 'mips.*el-.*' && echo -EB $(undef-all) $(predef-be)) cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += $(shell $(CC) -dumpmachine |grep -q 'mips.*el-.*' || echo -EL $(undef-all) $(predef-le)) +endif cflags-$(CONFIG_SB1XXX_CORELIS) += $(call cc-option,-mno-sched-prolog) \ -fno-omit-frame-pointer @@ -155,15 +165,11 @@ cflags-$(CONFIG_CPU_R4300) += -march=r4300 -Wa,--trap cflags-$(CONFIG_CPU_VR41XX) += -march=r4100 -Wa,--trap cflags-$(CONFIG_CPU_R4X00) += -march=r4600 -Wa,--trap cflags-$(CONFIG_CPU_TX49XX) += -march=r4600 -Wa,--trap -cflags-$(CONFIG_CPU_MIPS32_R1) += $(call cc-option,-march=mips32,-mips32 -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS32) \ - -Wa,-mips32 -Wa,--trap -cflags-$(CONFIG_CPU_MIPS32_R2) += $(call cc-option,-march=mips32r2,-mips32r2 -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS32) \ - -Wa,-mips32r2 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS32_R1) += -march=mips32 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS32_R2) += -march=mips32r2 -Wa,--trap cflags-$(CONFIG_CPU_MIPS32_R6) += -march=mips32r6 -Wa,--trap -modd-spreg -cflags-$(CONFIG_CPU_MIPS64_R1) += $(call cc-option,-march=mips64,-mips64 -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS64) \ - -Wa,-mips64 -Wa,--trap -cflags-$(CONFIG_CPU_MIPS64_R2) += $(call cc-option,-march=mips64r2,-mips64r2 -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS64) \ - -Wa,-mips64r2 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS64_R1) += -march=mips64 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS64_R2) += -march=mips64r2 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R6) += -march=mips64r6 -Wa,--trap cflags-$(CONFIG_CPU_R5000) += -march=r5000 -Wa,--trap cflags-$(CONFIG_CPU_R5432) += $(call cc-option,-march=r5400,-march=r5000) \ diff --git a/arch/mips/alchemy/board-gpr.c b/arch/mips/alchemy/board-gpr.c index fa75d75b5ba9..ddff9a02513d 100644 --- a/arch/mips/alchemy/board-gpr.c +++ b/arch/mips/alchemy/board-gpr.c @@ -34,6 +34,7 @@ #include <asm/bootinfo.h> #include <asm/idle.h> #include <asm/reboot.h> +#include <asm/setup.h> #include <asm/mach-au1x00/au1000.h> #include <asm/mach-au1x00/gpio-au1000.h> #include <prom.h> @@ -60,7 +61,7 @@ void __init prom_init(void) add_memory_region(0, memsize, BOOT_MEM_RAM); } -void prom_putchar(unsigned char c) +void prom_putchar(char c) { alchemy_uart_putchar(AU1000_UART0_PHYS_ADDR, c); } diff --git a/arch/mips/alchemy/board-mtx1.c b/arch/mips/alchemy/board-mtx1.c index aab55aaf3d62..d625e6f99ae7 100644 --- a/arch/mips/alchemy/board-mtx1.c +++ b/arch/mips/alchemy/board-mtx1.c @@ -31,6 +31,7 @@ #include <mtd/mtd-abi.h> #include <asm/bootinfo.h> #include <asm/reboot.h> +#include <asm/setup.h> #include <asm/mach-au1x00/au1000.h> #include <asm/mach-au1x00/gpio-au1000.h> #include <asm/mach-au1x00/au1xxx_eth.h> @@ -58,7 +59,7 @@ void __init prom_init(void) add_memory_region(0, memsize, BOOT_MEM_RAM); } -void prom_putchar(unsigned char c) +void prom_putchar(char c) { alchemy_uart_putchar(AU1000_UART0_PHYS_ADDR, c); } diff --git a/arch/mips/alchemy/board-xxs1500.c b/arch/mips/alchemy/board-xxs1500.c index 0fc53e08a894..5f05b8714385 100644 --- a/arch/mips/alchemy/board-xxs1500.c +++ b/arch/mips/alchemy/board-xxs1500.c @@ -29,6 +29,7 @@ #include <linux/pm.h> #include <asm/bootinfo.h> #include <asm/reboot.h> +#include <asm/setup.h> #include <asm/mach-au1x00/au1000.h> #include <prom.h> @@ -55,7 +56,7 @@ void __init prom_init(void) add_memory_region(0, memsize, BOOT_MEM_RAM); } -void prom_putchar(unsigned char c) +void prom_putchar(char c) { alchemy_uart_putchar(AU1000_UART0_PHYS_ADDR, c); } diff --git a/arch/mips/alchemy/devboards/platform.c b/arch/mips/alchemy/devboards/platform.c index 203854ddd1bb..8d4b65c3268a 100644 --- a/arch/mips/alchemy/devboards/platform.c +++ b/arch/mips/alchemy/devboards/platform.c @@ -14,6 +14,7 @@ #include <asm/bootinfo.h> #include <asm/idle.h> #include <asm/reboot.h> +#include <asm/setup.h> #include <asm/mach-au1x00/au1000.h> #include <asm/mach-db1x00/bcsr.h> @@ -36,7 +37,7 @@ void __init prom_init(void) add_memory_region(0, memsize, BOOT_MEM_RAM); } -void prom_putchar(unsigned char c) +void prom_putchar(char c) { if (alchemy_get_cputype() == ALCHEMY_CPU_AU1300) alchemy_uart_putchar(AU1300_UART2_PHYS_ADDR, c); diff --git a/arch/mips/ar7/clock.c b/arch/mips/ar7/clock.c index 0137656107a9..6b64fd96dba8 100644 --- a/arch/mips/ar7/clock.c +++ b/arch/mips/ar7/clock.c @@ -476,3 +476,32 @@ void __init ar7_init_clocks(void) /* adjust vbus clock rate */ vbus_clk.rate = bus_clk.rate / 2; } + +/* dummy functions, should not be called */ +long clk_round_rate(struct clk *clk, unsigned long rate) +{ + WARN_ON(clk); + return 0; +} +EXPORT_SYMBOL(clk_round_rate); + +int clk_set_rate(struct clk *clk, unsigned long rate) +{ + WARN_ON(clk); + return 0; +} +EXPORT_SYMBOL(clk_set_rate); + +int clk_set_parent(struct clk *clk, struct clk *parent) +{ + WARN_ON(clk); + return 0; +} +EXPORT_SYMBOL(clk_set_parent); + +struct clk *clk_get_parent(struct clk *clk) +{ + WARN_ON(clk); + return NULL; +} +EXPORT_SYMBOL(clk_get_parent); diff --git a/arch/mips/ar7/prom.c b/arch/mips/ar7/prom.c index dd53987a690f..2ec8d9ac91ec 100644 --- a/arch/mips/ar7/prom.c +++ b/arch/mips/ar7/prom.c @@ -25,6 +25,7 @@ #include <linux/string.h> #include <linux/io.h> #include <asm/bootinfo.h> +#include <asm/setup.h> #include <asm/mach-ar7/ar7.h> #include <asm/mach-ar7/prom.h> @@ -259,10 +260,9 @@ static inline void serial_out(int offset, int value) writel(value, (void *)PORT(offset)); } -int prom_putchar(char c) +void prom_putchar(char c) { while ((serial_in(UART_LSR) & UART_LSR_TEMT) == 0) ; serial_out(UART_TX, c); - return 1; } diff --git a/arch/mips/ath25/Kconfig b/arch/mips/ath25/Kconfig index 7070b4bcd01d..2c1dfd06c366 100644 --- a/arch/mips/ath25/Kconfig +++ b/arch/mips/ath25/Kconfig @@ -12,6 +12,7 @@ config SOC_AR2315 config PCI_AR2315 bool "Atheros AR2315 PCI controller support" depends on SOC_AR2315 + select ARCH_HAS_PHYS_TO_DMA select HW_HAS_PCI select PCI default y diff --git a/arch/mips/ath25/board.c b/arch/mips/ath25/board.c index 6d11ae581ea7..989e71015ee6 100644 --- a/arch/mips/ath25/board.c +++ b/arch/mips/ath25/board.c @@ -146,10 +146,10 @@ int __init ath25_find_config(phys_addr_t base, unsigned long size) pr_info("Fixing up empty mac addresses\n"); config->reset_config_gpio = 0xffff; config->sys_led_gpio = 0xffff; - random_ether_addr(config->wlan0_mac); + eth_random_addr(config->wlan0_mac); config->wlan0_mac[0] &= ~0x06; - random_ether_addr(config->enet0_mac); - random_ether_addr(config->enet1_mac); + eth_random_addr(config->enet0_mac); + eth_random_addr(config->enet1_mac); } } diff --git a/arch/mips/ath25/early_printk.c b/arch/mips/ath25/early_printk.c index 36035b628161..d534761e9cda 100644 --- a/arch/mips/ath25/early_printk.c +++ b/arch/mips/ath25/early_printk.c @@ -9,6 +9,7 @@ #include <linux/mm.h> #include <linux/io.h> #include <linux/serial_reg.h> +#include <asm/setup.h> #include "devices.h" #include "ar2315_regs.h" @@ -25,7 +26,7 @@ static inline unsigned char prom_uart_rr(void __iomem *base, unsigned reg) return __raw_readl(base + 4 * reg); } -void prom_putchar(unsigned char ch) +void prom_putchar(char ch) { static void __iomem *base; @@ -38,7 +39,7 @@ void prom_putchar(unsigned char ch) while ((prom_uart_rr(base, UART_LSR) & UART_LSR_THRE) == 0) ; - prom_uart_wr(base, UART_TX, ch); + prom_uart_wr(base, UART_TX, (unsigned char)ch); while ((prom_uart_rr(base, UART_LSR) & UART_LSR_THRE) == 0) ; } diff --git a/arch/mips/ath79/clock.c b/arch/mips/ath79/clock.c index 6b1000b6a6a6..cf9158e3c2d9 100644 --- a/arch/mips/ath79/clock.c +++ b/arch/mips/ath79/clock.c @@ -355,6 +355,91 @@ static void __init ar934x_clocks_init(void) iounmap(dpll_base); } +static void __init qca953x_clocks_init(void) +{ + unsigned long ref_rate; + unsigned long cpu_rate; + unsigned long ddr_rate; + unsigned long ahb_rate; + u32 pll, out_div, ref_div, nint, frac, clk_ctrl, postdiv; + u32 cpu_pll, ddr_pll; + u32 bootstrap; + + bootstrap = ath79_reset_rr(QCA953X_RESET_REG_BOOTSTRAP); + if (bootstrap & QCA953X_BOOTSTRAP_REF_CLK_40) + ref_rate = 40 * 1000 * 1000; + else + ref_rate = 25 * 1000 * 1000; + + pll = ath79_pll_rr(QCA953X_PLL_CPU_CONFIG_REG); + out_div = (pll >> QCA953X_PLL_CPU_CONFIG_OUTDIV_SHIFT) & + QCA953X_PLL_CPU_CONFIG_OUTDIV_MASK; + ref_div = (pll >> QCA953X_PLL_CPU_CONFIG_REFDIV_SHIFT) & + QCA953X_PLL_CPU_CONFIG_REFDIV_MASK; + nint = (pll >> QCA953X_PLL_CPU_CONFIG_NINT_SHIFT) & + QCA953X_PLL_CPU_CONFIG_NINT_MASK; + frac = (pll >> QCA953X_PLL_CPU_CONFIG_NFRAC_SHIFT) & + QCA953X_PLL_CPU_CONFIG_NFRAC_MASK; + + cpu_pll = nint * ref_rate / ref_div; + cpu_pll += frac * (ref_rate >> 6) / ref_div; + cpu_pll /= (1 << out_div); + + pll = ath79_pll_rr(QCA953X_PLL_DDR_CONFIG_REG); + out_div = (pll >> QCA953X_PLL_DDR_CONFIG_OUTDIV_SHIFT) & + QCA953X_PLL_DDR_CONFIG_OUTDIV_MASK; + ref_div = (pll >> QCA953X_PLL_DDR_CONFIG_REFDIV_SHIFT) & + QCA953X_PLL_DDR_CONFIG_REFDIV_MASK; + nint = (pll >> QCA953X_PLL_DDR_CONFIG_NINT_SHIFT) & + QCA953X_PLL_DDR_CONFIG_NINT_MASK; + frac = (pll >> QCA953X_PLL_DDR_CONFIG_NFRAC_SHIFT) & + QCA953X_PLL_DDR_CONFIG_NFRAC_MASK; + + ddr_pll = nint * ref_rate / ref_div; + ddr_pll += frac * (ref_rate >> 6) / (ref_div << 4); + ddr_pll /= (1 << out_div); + + clk_ctrl = ath79_pll_rr(QCA953X_PLL_CLK_CTRL_REG); + + postdiv = (clk_ctrl >> QCA953X_PLL_CLK_CTRL_CPU_POST_DIV_SHIFT) & + QCA953X_PLL_CLK_CTRL_CPU_POST_DIV_MASK; + + if (clk_ctrl & QCA953X_PLL_CLK_CTRL_CPU_PLL_BYPASS) + cpu_rate = ref_rate; + else if (clk_ctrl & QCA953X_PLL_CLK_CTRL_CPUCLK_FROM_CPUPLL) + cpu_rate = cpu_pll / (postdiv + 1); + else + cpu_rate = ddr_pll / (postdiv + 1); + + postdiv = (clk_ctrl >> QCA953X_PLL_CLK_CTRL_DDR_POST_DIV_SHIFT) & + QCA953X_PLL_CLK_CTRL_DDR_POST_DIV_MASK; + + if (clk_ctrl & QCA953X_PLL_CLK_CTRL_DDR_PLL_BYPASS) + ddr_rate = ref_rate; + else if (clk_ctrl & QCA953X_PLL_CLK_CTRL_DDRCLK_FROM_DDRPLL) + ddr_rate = ddr_pll / (postdiv + 1); + else + ddr_rate = cpu_pll / (postdiv + 1); + + postdiv = (clk_ctrl >> QCA953X_PLL_CLK_CTRL_AHB_POST_DIV_SHIFT) & + QCA953X_PLL_CLK_CTRL_AHB_POST_DIV_MASK; + + if (clk_ctrl & QCA953X_PLL_CLK_CTRL_AHB_PLL_BYPASS) + ahb_rate = ref_rate; + else if (clk_ctrl & QCA953X_PLL_CLK_CTRL_AHBCLK_FROM_DDRPLL) + ahb_rate = ddr_pll / (postdiv + 1); + else + ahb_rate = cpu_pll / (postdiv + 1); + + ath79_add_sys_clkdev("ref", ref_rate); + ath79_add_sys_clkdev("cpu", cpu_rate); + ath79_add_sys_clkdev("ddr", ddr_rate); + ath79_add_sys_clkdev("ahb", ahb_rate); + + clk_add_alias("wdt", NULL, "ref", NULL); + clk_add_alias("uart", NULL, "ref", NULL); +} + static void __init qca955x_clocks_init(void) { unsigned long ref_rate; @@ -440,6 +525,110 @@ static void __init qca955x_clocks_init(void) clk_add_alias("uart", NULL, "ref", NULL); } +static void __init qca956x_clocks_init(void) +{ + unsigned long ref_rate; + unsigned long cpu_rate; + unsigned long ddr_rate; + unsigned long ahb_rate; + u32 pll, out_div, ref_div, nint, hfrac, lfrac, clk_ctrl, postdiv; + u32 cpu_pll, ddr_pll; + u32 bootstrap; + + /* + * QCA956x timer init workaround has to be applied right before setting + * up the clock. Else, there will be no jiffies + */ + u32 misc; + + misc = ath79_reset_rr(AR71XX_RESET_REG_MISC_INT_ENABLE); + misc |= MISC_INT_MIPS_SI_TIMERINT_MASK; + ath79_reset_wr(AR71XX_RESET_REG_MISC_INT_ENABLE, misc); + + bootstrap = ath79_reset_rr(QCA956X_RESET_REG_BOOTSTRAP); + if (bootstrap & QCA956X_BOOTSTRAP_REF_CLK_40) + ref_rate = 40 * 1000 * 1000; + else + ref_rate = 25 * 1000 * 1000; + + pll = ath79_pll_rr(QCA956X_PLL_CPU_CONFIG_REG); + out_div = (pll >> QCA956X_PLL_CPU_CONFIG_OUTDIV_SHIFT) & + QCA956X_PLL_CPU_CONFIG_OUTDIV_MASK; + ref_div = (pll >> QCA956X_PLL_CPU_CONFIG_REFDIV_SHIFT) & + QCA956X_PLL_CPU_CONFIG_REFDIV_MASK; + + pll = ath79_pll_rr(QCA956X_PLL_CPU_CONFIG1_REG); + nint = (pll >> QCA956X_PLL_CPU_CONFIG1_NINT_SHIFT) & + QCA956X_PLL_CPU_CONFIG1_NINT_MASK; + hfrac = (pll >> QCA956X_PLL_CPU_CONFIG1_NFRAC_H_SHIFT) & + QCA956X_PLL_CPU_CONFIG1_NFRAC_H_MASK; + lfrac = (pll >> QCA956X_PLL_CPU_CONFIG1_NFRAC_L_SHIFT) & + QCA956X_PLL_CPU_CONFIG1_NFRAC_L_MASK; + + cpu_pll = nint * ref_rate / ref_div; + cpu_pll += (lfrac * ref_rate) / ((ref_div * 25) << 13); + cpu_pll += (hfrac >> 13) * ref_rate / ref_div; + cpu_pll /= (1 << out_div); + + pll = ath79_pll_rr(QCA956X_PLL_DDR_CONFIG_REG); + out_div = (pll >> QCA956X_PLL_DDR_CONFIG_OUTDIV_SHIFT) & + QCA956X_PLL_DDR_CONFIG_OUTDIV_MASK; + ref_div = (pll >> QCA956X_PLL_DDR_CONFIG_REFDIV_SHIFT) & + QCA956X_PLL_DDR_CONFIG_REFDIV_MASK; + pll = ath79_pll_rr(QCA956X_PLL_DDR_CONFIG1_REG); + nint = (pll >> QCA956X_PLL_DDR_CONFIG1_NINT_SHIFT) & + QCA956X_PLL_DDR_CONFIG1_NINT_MASK; + hfrac = (pll >> QCA956X_PLL_DDR_CONFIG1_NFRAC_H_SHIFT) & + QCA956X_PLL_DDR_CONFIG1_NFRAC_H_MASK; + lfrac = (pll >> QCA956X_PLL_DDR_CONFIG1_NFRAC_L_SHIFT) & + QCA956X_PLL_DDR_CONFIG1_NFRAC_L_MASK; + + ddr_pll = nint * ref_rate / ref_div; + ddr_pll += (lfrac * ref_rate) / ((ref_div * 25) << 13); + ddr_pll += (hfrac >> 13) * ref_rate / ref_div; + ddr_pll /= (1 << out_div); + + clk_ctrl = ath79_pll_rr(QCA956X_PLL_CLK_CTRL_REG); + + postdiv = (clk_ctrl >> QCA956X_PLL_CLK_CTRL_CPU_POST_DIV_SHIFT) & + QCA956X_PLL_CLK_CTRL_CPU_POST_DIV_MASK; + + if (clk_ctrl & QCA956X_PLL_CLK_CTRL_CPU_PLL_BYPASS) + cpu_rate = ref_rate; + else if (clk_ctrl & QCA956X_PLL_CLK_CTRL_CPU_DDRCLK_FROM_CPUPLL) + cpu_rate = ddr_pll / (postdiv + 1); + else + cpu_rate = cpu_pll / (postdiv + 1); + + postdiv = (clk_ctrl >> QCA956X_PLL_CLK_CTRL_DDR_POST_DIV_SHIFT) & + QCA956X_PLL_CLK_CTRL_DDR_POST_DIV_MASK; + + if (clk_ctrl & QCA956X_PLL_CLK_CTRL_DDR_PLL_BYPASS) + ddr_rate = ref_rate; + else if (clk_ctrl & QCA956X_PLL_CLK_CTRL_CPU_DDRCLK_FROM_DDRPLL) + ddr_rate = cpu_pll / (postdiv + 1); + else + ddr_rate = ddr_pll / (postdiv + 1); + + postdiv = (clk_ctrl >> QCA956X_PLL_CLK_CTRL_AHB_POST_DIV_SHIFT) & + QCA956X_PLL_CLK_CTRL_AHB_POST_DIV_MASK; + + if (clk_ctrl & QCA956X_PLL_CLK_CTRL_AHB_PLL_BYPASS) + ahb_rate = ref_rate; + else if (clk_ctrl & QCA956X_PLL_CLK_CTRL_AHBCLK_FROM_DDRPLL) + ahb_rate = ddr_pll / (postdiv + 1); + else + ahb_rate = cpu_pll / (postdiv + 1); + + ath79_add_sys_clkdev("ref", ref_rate); + ath79_add_sys_clkdev("cpu", cpu_rate); + ath79_add_sys_clkdev("ddr", ddr_rate); + ath79_add_sys_clkdev("ahb", ahb_rate); + + clk_add_alias("wdt", NULL, "ref", NULL); + clk_add_alias("uart", NULL, "ref", NULL); +} + void __init ath79_clocks_init(void) { if (soc_is_ar71xx()) @@ -450,8 +639,12 @@ void __init ath79_clocks_init(void) ar933x_clocks_init(); else if (soc_is_ar934x()) ar934x_clocks_init(); + else if (soc_is_qca953x()) + qca953x_clocks_init(); else if (soc_is_qca955x()) qca955x_clocks_init(); + else if (soc_is_qca956x() || soc_is_tp9343()) + qca956x_clocks_init(); else BUG(); } diff --git a/arch/mips/ath79/common.c b/arch/mips/ath79/common.c index 10a405d593df..cd6055f9e7a0 100644 --- a/arch/mips/ath79/common.c +++ b/arch/mips/ath79/common.c @@ -58,7 +58,7 @@ EXPORT_SYMBOL_GPL(ath79_ddr_ctrl_init); void ath79_ddr_wb_flush(u32 reg) { - void __iomem *flush_reg = ath79_ddr_wb_flush_base + reg; + void __iomem *flush_reg = ath79_ddr_wb_flush_base + (reg * 4); /* Flush the DDR write buffer. */ __raw_writel(0x1, flush_reg); @@ -103,8 +103,12 @@ void ath79_device_reset_set(u32 mask) reg = AR933X_RESET_REG_RESET_MODULE; else if (soc_is_ar934x()) reg = AR934X_RESET_REG_RESET_MODULE; + else if (soc_is_qca953x()) + reg = QCA953X_RESET_REG_RESET_MODULE; else if (soc_is_qca955x()) reg = QCA955X_RESET_REG_RESET_MODULE; + else if (soc_is_qca956x() || soc_is_tp9343()) + reg = QCA956X_RESET_REG_RESET_MODULE; else BUG(); @@ -131,8 +135,12 @@ void ath79_device_reset_clear(u32 mask) reg = AR933X_RESET_REG_RESET_MODULE; else if (soc_is_ar934x()) reg = AR934X_RESET_REG_RESET_MODULE; + else if (soc_is_qca953x()) + reg = QCA953X_RESET_REG_RESET_MODULE; else if (soc_is_qca955x()) reg = QCA955X_RESET_REG_RESET_MODULE; + else if (soc_is_qca956x() || soc_is_tp9343()) + reg = QCA956X_RESET_REG_RESET_MODULE; else BUG(); diff --git a/arch/mips/ath79/early_printk.c b/arch/mips/ath79/early_printk.c index d1adc59af5bf..4b1063117ef7 100644 --- a/arch/mips/ath79/early_printk.c +++ b/arch/mips/ath79/early_printk.c @@ -13,12 +13,13 @@ #include <linux/errno.h> #include <linux/serial_reg.h> #include <asm/addrspace.h> +#include <asm/setup.h> #include <asm/mach-ath79/ath79.h> #include <asm/mach-ath79/ar71xx_regs.h> #include <asm/mach-ath79/ar933x_uart.h> -static void (*_prom_putchar) (unsigned char); +static void (*_prom_putchar)(char); static inline void prom_putchar_wait(void __iomem *reg, u32 mask, u32 val) { @@ -33,31 +34,72 @@ static inline void prom_putchar_wait(void __iomem *reg, u32 mask, u32 val) #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) -static void prom_putchar_ar71xx(unsigned char ch) +static void prom_putchar_ar71xx(char ch) { void __iomem *base = (void __iomem *)(KSEG1ADDR(AR71XX_UART_BASE)); prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); - __raw_writel(ch, base + UART_TX * 4); + __raw_writel((unsigned char)ch, base + UART_TX * 4); prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); } -static void prom_putchar_ar933x(unsigned char ch) +static void prom_putchar_ar933x(char ch) { void __iomem *base = (void __iomem *)(KSEG1ADDR(AR933X_UART_BASE)); prom_putchar_wait(base + AR933X_UART_DATA_REG, AR933X_UART_DATA_TX_CSR, AR933X_UART_DATA_TX_CSR); - __raw_writel(AR933X_UART_DATA_TX_CSR | ch, base + AR933X_UART_DATA_REG); + __raw_writel(AR933X_UART_DATA_TX_CSR | (unsigned char)ch, + base + AR933X_UART_DATA_REG); prom_putchar_wait(base + AR933X_UART_DATA_REG, AR933X_UART_DATA_TX_CSR, AR933X_UART_DATA_TX_CSR); } -static void prom_putchar_dummy(unsigned char ch) +static void prom_putchar_dummy(char ch) { /* nothing to do */ } +static void prom_enable_uart(u32 id) +{ + void __iomem *gpio_base; + u32 uart_en; + u32 t; + + switch (id) { + case REV_ID_MAJOR_AR71XX: + uart_en = AR71XX_GPIO_FUNC_UART_EN; + break; + + case REV_ID_MAJOR_AR7240: + case REV_ID_MAJOR_AR7241: + case REV_ID_MAJOR_AR7242: + uart_en = AR724X_GPIO_FUNC_UART_EN; + break; + + case REV_ID_MAJOR_AR913X: + uart_en = AR913X_GPIO_FUNC_UART_EN; + break; + + case REV_ID_MAJOR_AR9330: + case REV_ID_MAJOR_AR9331: + uart_en = AR933X_GPIO_FUNC_UART_EN; + break; + + case REV_ID_MAJOR_AR9341: + case REV_ID_MAJOR_AR9342: + case REV_ID_MAJOR_AR9344: + /* TODO */ + default: + return; + } + + gpio_base = (void __iomem *)KSEG1ADDR(AR71XX_GPIO_BASE); + t = __raw_readl(gpio_base + AR71XX_GPIO_REG_FUNC); + t |= uart_en; + __raw_writel(t, gpio_base + AR71XX_GPIO_REG_FUNC); +} + static void prom_putchar_init(void) { void __iomem *base; @@ -76,8 +118,12 @@ static void prom_putchar_init(void) case REV_ID_MAJOR_AR9341: case REV_ID_MAJOR_AR9342: case REV_ID_MAJOR_AR9344: + case REV_ID_MAJOR_QCA9533: + case REV_ID_MAJOR_QCA9533_V2: case REV_ID_MAJOR_QCA9556: case REV_ID_MAJOR_QCA9558: + case REV_ID_MAJOR_TP9343: + case REV_ID_MAJOR_QCA956X: _prom_putchar = prom_putchar_ar71xx; break; @@ -88,11 +134,13 @@ static void prom_putchar_init(void) default: _prom_putchar = prom_putchar_dummy; - break; + return; } + + prom_enable_uart(id); } -void prom_putchar(unsigned char ch) +void prom_putchar(char ch) { if (!_prom_putchar) prom_putchar_init(); diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index f206dafbb0a3..4c7a93f4039a 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -40,6 +40,7 @@ static char ath79_sys_type[ATH79_SYS_TYPE_LEN]; static void ath79_restart(char *command) { + local_irq_disable(); ath79_device_reset_set(AR71XX_RESET_FULL_CHIP); for (;;) if (cpu_wait) @@ -59,6 +60,7 @@ static void __init ath79_detect_sys_type(void) u32 major; u32 minor; u32 rev = 0; + u32 ver = 1; id = ath79_reset_rr(AR71XX_RESET_REG_REV_ID); major = id & REV_ID_MAJOR_MASK; @@ -151,6 +153,17 @@ static void __init ath79_detect_sys_type(void) rev = id & AR934X_REV_ID_REVISION_MASK; break; + case REV_ID_MAJOR_QCA9533_V2: + ver = 2; + ath79_soc_rev = 2; + /* drop through */ + + case REV_ID_MAJOR_QCA9533: + ath79_soc = ATH79_SOC_QCA9533; + chip = "9533"; + rev = id & QCA953X_REV_ID_REVISION_MASK; + break; + case REV_ID_MAJOR_QCA9556: ath79_soc = ATH79_SOC_QCA9556; chip = "9556"; @@ -163,14 +176,30 @@ static void __init ath79_detect_sys_type(void) rev = id & QCA955X_REV_ID_REVISION_MASK; break; + case REV_ID_MAJOR_QCA956X: + ath79_soc = ATH79_SOC_QCA956X; + chip = "956X"; + rev = id & QCA956X_REV_ID_REVISION_MASK; + break; + + case REV_ID_MAJOR_TP9343: + ath79_soc = ATH79_SOC_TP9343; + chip = "9343"; + rev = id & QCA956X_REV_ID_REVISION_MASK; + break; + default: panic("ath79: unknown SoC, id:0x%08x", id); } - ath79_soc_rev = rev; + if (ver == 1) + ath79_soc_rev = rev; - if (soc_is_qca955x()) - sprintf(ath79_sys_type, "Qualcomm Atheros QCA%s rev %u", + if (soc_is_qca953x() || soc_is_qca955x() || soc_is_qca956x()) + sprintf(ath79_sys_type, "Qualcomm Atheros QCA%s ver %u rev %u", + chip, ver, rev); + else if (soc_is_tp9343()) + sprintf(ath79_sys_type, "Qualcomm Atheros TP%s rev %u", chip, rev); else sprintf(ath79_sys_type, "Atheros AR%s rev %u", chip, rev); diff --git a/arch/mips/bcm47xx/setup.c b/arch/mips/bcm47xx/setup.c index 8c9cbf13d32a..6054d49e608e 100644 --- a/arch/mips/bcm47xx/setup.c +++ b/arch/mips/bcm47xx/setup.c @@ -212,12 +212,6 @@ static int __init bcm47xx_cpu_fixes(void) */ if (bcm47xx_bus.bcma.bus.chipinfo.id == BCMA_CHIP_ID_BCM4706) cpu_wait = NULL; - - /* - * BCM47XX Erratum "R10: PCIe Transactions Periodically Fail" - * Enable ExternalSync for sync instruction to take effect - */ - set_c0_config7(MIPS_CONF7_ES); break; #endif } diff --git a/arch/mips/bcm63xx/early_printk.c b/arch/mips/bcm63xx/early_printk.c index 6092226a6d76..9e9ec27c282f 100644 --- a/arch/mips/bcm63xx/early_printk.c +++ b/arch/mips/bcm63xx/early_printk.c @@ -8,6 +8,7 @@ #include <bcm63xx_io.h> #include <linux/serial_bcm63xx.h> +#include <asm/setup.h> static void wait_xfered(void) { diff --git a/arch/mips/bmips/dma.c b/arch/mips/bmips/dma.c index 6dec30842b2f..3d13c77c125f 100644 --- a/arch/mips/bmips/dma.c +++ b/arch/mips/bmips/dma.c @@ -17,7 +17,7 @@ #include <linux/printk.h> #include <linux/slab.h> #include <linux/types.h> -#include <dma-coherence.h> +#include <asm/bmips.h> /* * BCM338x has configurable address translation windows which allow the @@ -40,7 +40,7 @@ static struct bmips_dma_range *bmips_dma_ranges; #define FLUSH_RAC 0x100 -static dma_addr_t bmips_phys_to_dma(struct device *dev, phys_addr_t pa) +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t pa) { struct bmips_dma_range *r; @@ -52,17 +52,7 @@ static dma_addr_t bmips_phys_to_dma(struct device *dev, phys_addr_t pa) return pa; } -dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, size_t size) -{ - return bmips_phys_to_dma(dev, virt_to_phys(addr)); -} - -dma_addr_t plat_map_dma_mem_page(struct device *dev, struct page *page) -{ - return bmips_phys_to_dma(dev, page_to_phys(page)); -} - -unsigned long plat_dma_addr_to_phys(struct device *dev, dma_addr_t dma_addr) +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) { struct bmips_dma_range *r; @@ -74,6 +64,22 @@ unsigned long plat_dma_addr_to_phys(struct device *dev, dma_addr_t dma_addr) return dma_addr; } +void arch_sync_dma_for_cpu_all(struct device *dev) +{ + void __iomem *cbr = BMIPS_GET_CBR(); + u32 cfg; + + if (boot_cpu_type() != CPU_BMIPS3300 && + boot_cpu_type() != CPU_BMIPS4350 && + boot_cpu_type() != CPU_BMIPS4380) + return; + + /* Flush stale data out of the readahead cache */ + cfg = __raw_readl(cbr + BMIPS_RAC_CONFIG); + __raw_writel(cfg | 0x100, cbr + BMIPS_RAC_CONFIG); + __raw_readl(cbr + BMIPS_RAC_CONFIG); +} + static int __init bmips_init_dma_ranges(void) { struct device_node *np = diff --git a/arch/mips/bmips/setup.c b/arch/mips/bmips/setup.c index 3b6f687f177c..231fc5ce375e 100644 --- a/arch/mips/bmips/setup.c +++ b/arch/mips/bmips/setup.c @@ -202,13 +202,6 @@ void __init device_tree_init(void) of_node_put(np); } -int __init plat_of_setup(void) -{ - return __dt_register_buses("simple-bus", NULL); -} - -arch_initcall(plat_of_setup); - static int __init plat_dev_init(void) { of_clk_init(NULL); diff --git a/arch/mips/boot/Makefile b/arch/mips/boot/Makefile index c22da16d67b8..35704c28a28b 100644 --- a/arch/mips/boot/Makefile +++ b/arch/mips/boot/Makefile @@ -105,28 +105,29 @@ $(obj)/uImage: $(obj)/uImage.$(suffix-y) # Flattened Image Tree (.itb) images # -targets += vmlinux.itb -targets += vmlinux.gz.itb -targets += vmlinux.bz2.itb -targets += vmlinux.lzma.itb -targets += vmlinux.lzo.itb - ifeq ($(ADDR_BITS),32) - itb_addr_cells = 1 +itb_addr_cells = 1 endif ifeq ($(ADDR_BITS),64) - itb_addr_cells = 2 +itb_addr_cells = 2 endif +targets += vmlinux.its.S + quiet_cmd_its_cat = CAT $@ - cmd_its_cat = cat $^ >$@ + cmd_its_cat = cat $(filter-out $(PHONY), $^) >$@ -$(obj)/vmlinux.its.S: $(addprefix $(srctree)/arch/mips/$(PLATFORM)/,$(ITS_INPUTS)) +$(obj)/vmlinux.its.S: $(addprefix $(srctree)/arch/mips/$(PLATFORM)/,$(ITS_INPUTS)) FORCE $(call if_changed,its_cat) +targets += vmlinux.its +targets += vmlinux.gz.its +targets += vmlinux.bz2.its +targets += vmlinux.lzmo.its +targets += vmlinux.lzo.its + quiet_cmd_cpp_its_S = ITS $@ - cmd_cpp_its_S = $(CPP) $(cpp_flags) -P -C -o $@ $< \ - -D__ASSEMBLY__ \ + cmd_cpp_its_S = $(CPP) -P -C -o $@ $< \ -DKERNEL_NAME="\"Linux $(KERNELRELEASE)\"" \ -DVMLINUX_BINARY="\"$(3)\"" \ -DVMLINUX_COMPRESSION="\"$(2)\"" \ @@ -136,19 +137,25 @@ quiet_cmd_cpp_its_S = ITS $@ -DADDR_CELLS=$(itb_addr_cells) $(obj)/vmlinux.its: $(obj)/vmlinux.its.S $(VMLINUX) FORCE - $(call if_changed_dep,cpp_its_S,none,vmlinux.bin) + $(call if_changed,cpp_its_S,none,vmlinux.bin) $(obj)/vmlinux.gz.its: $(obj)/vmlinux.its.S $(VMLINUX) FORCE - $(call if_changed_dep,cpp_its_S,gzip,vmlinux.bin.gz) + $(call if_changed,cpp_its_S,gzip,vmlinux.bin.gz) $(obj)/vmlinux.bz2.its: $(obj)/vmlinux.its.S $(VMLINUX) FORCE - $(call if_changed_dep,cpp_its_S,bzip2,vmlinux.bin.bz2) + $(call if_changed,cpp_its_S,bzip2,vmlinux.bin.bz2) $(obj)/vmlinux.lzma.its: $(obj)/vmlinux.its.S $(VMLINUX) FORCE - $(call if_changed_dep,cpp_its_S,lzma,vmlinux.bin.lzma) + $(call if_changed,cpp_its_S,lzma,vmlinux.bin.lzma) $(obj)/vmlinux.lzo.its: $(obj)/vmlinux.its.S $(VMLINUX) FORCE - $(call if_changed_dep,cpp_its_S,lzo,vmlinux.bin.lzo) + $(call if_changed,cpp_its_S,lzo,vmlinux.bin.lzo) + +targets += vmlinux.itb +targets += vmlinux.gz.itb +targets += vmlinux.bz2.itb +targets += vmlinux.lzma.itb +targets += vmlinux.lzo.itb quiet_cmd_itb-image = ITB $@ cmd_itb-image = \ @@ -162,14 +169,5 @@ quiet_cmd_itb-image = ITB $@ $(obj)/vmlinux.itb: $(obj)/vmlinux.its $(obj)/vmlinux.bin FORCE $(call if_changed,itb-image,$<) -$(obj)/vmlinux.gz.itb: $(obj)/vmlinux.gz.its $(obj)/vmlinux.bin.gz FORCE - $(call if_changed,itb-image,$<) - -$(obj)/vmlinux.bz2.itb: $(obj)/vmlinux.bz2.its $(obj)/vmlinux.bin.bz2 FORCE - $(call if_changed,itb-image,$<) - -$(obj)/vmlinux.lzma.itb: $(obj)/vmlinux.lzma.its $(obj)/vmlinux.bin.lzma FORCE - $(call if_changed,itb-image,$<) - -$(obj)/vmlinux.lzo.itb: $(obj)/vmlinux.lzo.its $(obj)/vmlinux.bin.lzo FORCE +$(obj)/vmlinux.%.itb: $(obj)/vmlinux.%.its $(obj)/vmlinux.bin.% FORCE $(call if_changed,itb-image,$<) diff --git a/arch/mips/boot/compressed/uart-prom.c b/arch/mips/boot/compressed/uart-prom.c index d6f0fee0a151..a8a0a32e05d1 100644 --- a/arch/mips/boot/compressed/uart-prom.c +++ b/arch/mips/boot/compressed/uart-prom.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 - -extern void prom_putchar(unsigned char ch); +#include <asm/setup.h> void putc(char c) { diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi index aa4e8f75ff5d..ce93d57f1b4d 100644 --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi @@ -155,6 +155,25 @@ }; }; + spi_gpio { + compatible = "spi-gpio"; + #address-cells = <1>; + #size-cells = <0>; + num-chipselects = <2>; + + gpio-miso = <&gpe 14 0>; + gpio-sck = <&gpe 15 0>; + gpio-mosi = <&gpe 17 0>; + cs-gpios = <&gpe 16 0 + &gpe 18 0>; + + spidev@0 { + compatible = "spidev"; + reg = <0>; + spi-max-frequency = <1000000>; + }; + }; + uart0: serial@10030000 { compatible = "ingenic,jz4780-uart"; reg = <0x10030000 0x100>; diff --git a/arch/mips/boot/dts/mscc/Makefile b/arch/mips/boot/dts/mscc/Makefile index 3c6aed9f5439..9a9bb7ea0503 100644 --- a/arch/mips/boot/dts/mscc/Makefile +++ b/arch/mips/boot/dts/mscc/Makefile @@ -1,3 +1,3 @@ -dtb-$(CONFIG_LEGACY_BOARD_OCELOT) += ocelot_pcb123.dtb +dtb-$(CONFIG_MSCC_OCELOT) += ocelot_pcb123.dtb obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/mips/boot/dts/mscc/ocelot.dtsi b/arch/mips/boot/dts/mscc/ocelot.dtsi index 4f33dbc67348..f7eb612b46ba 100644 --- a/arch/mips/boot/dts/mscc/ocelot.dtsi +++ b/arch/mips/boot/dts/mscc/ocelot.dtsi @@ -91,6 +91,17 @@ status = "disabled"; }; + spi: spi@101000 { + compatible = "mscc,ocelot-spi", "snps,dw-apb-ssi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x101000 0x100>, <0x3c 0x18>; + interrupts = <9>; + clocks = <&ahb_clk>; + + status = "disabled"; + }; + switch@1010000 { compatible = "mscc,vsc7514-switch"; reg = <0x1010000 0x10000>, @@ -168,6 +179,9 @@ gpio-controller; #gpio-cells = <2>; gpio-ranges = <&gpio 0 0 22>; + interrupt-controller; + interrupts = <13>; + #interrupt-cells = <2>; uart_pins: uart-pins { pins = "GPIO_6", "GPIO_7"; @@ -178,13 +192,18 @@ pins = "GPIO_12", "GPIO_13"; function = "uart2"; }; + + miim1: miim1 { + pins = "GPIO_14", "GPIO_15"; + function = "miim1"; + }; }; mdio0: mdio@107009c { #address-cells = <1>; #size-cells = <0>; compatible = "mscc,ocelot-miim"; - reg = <0x107009c 0x36>, <0x10700f0 0x8>; + reg = <0x107009c 0x24>, <0x10700f0 0x8>; interrupts = <14>; status = "disabled"; @@ -201,5 +220,16 @@ reg = <3>; }; }; + + mdio1: mdio@10700c0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "mscc,ocelot-miim"; + reg = <0x10700c0 0x24>; + interrupts = <15>; + pinctrl-names = "default"; + pinctrl-0 = <&miim1>; + status = "disabled"; + }; }; }; diff --git a/arch/mips/boot/dts/mscc/ocelot_pcb123.dts b/arch/mips/boot/dts/mscc/ocelot_pcb123.dts index 4ccd65379059..2266027759f9 100644 --- a/arch/mips/boot/dts/mscc/ocelot_pcb123.dts +++ b/arch/mips/boot/dts/mscc/ocelot_pcb123.dts @@ -26,6 +26,16 @@ status = "okay"; }; +&spi { + status = "okay"; + + flash@0 { + compatible = "macronix,mx25l25635f", "jedec,spi-nor"; + spi-max-frequency = <20000000>; + reg = <0>; + }; +}; + &mdio0 { status = "okay"; }; diff --git a/arch/mips/boot/dts/qca/ar9132.dtsi b/arch/mips/boot/dts/qca/ar9132.dtsi index 1fe561c5f90e..61dcfa5b6ca7 100644 --- a/arch/mips/boot/dts/qca/ar9132.dtsi +++ b/arch/mips/boot/dts/qca/ar9132.dtsi @@ -161,7 +161,7 @@ usb_phy: usb-phy { compatible = "qca,ar7100-usb-phy"; - reset-names = "usb-phy", "usb-suspend-override"; + reset-names = "phy", "suspend-override"; resets = <&rst 4>, <&rst 3>; #phy-cells = <0>; diff --git a/arch/mips/boot/dts/qca/ar9132_tl_wr1043nd_v1.dts b/arch/mips/boot/dts/qca/ar9132_tl_wr1043nd_v1.dts index 3931033e47c8..7fccf6357225 100644 --- a/arch/mips/boot/dts/qca/ar9132_tl_wr1043nd_v1.dts +++ b/arch/mips/boot/dts/qca/ar9132_tl_wr1043nd_v1.dts @@ -22,11 +22,10 @@ }; gpio-keys { - compatible = "gpio-keys-polled"; + compatible = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; - poll-interval = <20>; button@0 { label = "reset"; linux,code = <KEY_RESTART>; diff --git a/arch/mips/boot/dts/qca/ar9331.dtsi b/arch/mips/boot/dts/qca/ar9331.dtsi index efd5f0722206..2bae201aa365 100644 --- a/arch/mips/boot/dts/qca/ar9331.dtsi +++ b/arch/mips/boot/dts/qca/ar9331.dtsi @@ -146,7 +146,7 @@ usb_phy: usb-phy { compatible = "qca,ar7100-usb-phy"; - reset-names = "usb-phy", "usb-suspend-override"; + reset-names = "phy", "suspend-override"; resets = <&rst 4>, <&rst 3>; #phy-cells = <0>; diff --git a/arch/mips/boot/dts/qca/ar9331_dpt_module.dts b/arch/mips/boot/dts/qca/ar9331_dpt_module.dts index d4e4502daaa8..e7af2cf5f4c1 100644 --- a/arch/mips/boot/dts/qca/ar9331_dpt_module.dts +++ b/arch/mips/boot/dts/qca/ar9331_dpt_module.dts @@ -29,11 +29,10 @@ }; }; - gpio-keys-polled { - compatible = "gpio-keys-polled"; + gpio-keys { + compatible = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; - poll-interval = <100>; button@0 { label = "reset"; diff --git a/arch/mips/boot/dts/qca/ar9331_dragino_ms14.dts b/arch/mips/boot/dts/qca/ar9331_dragino_ms14.dts index 4f95ccf17c4c..d38aa73f1a2e 100644 --- a/arch/mips/boot/dts/qca/ar9331_dragino_ms14.dts +++ b/arch/mips/boot/dts/qca/ar9331_dragino_ms14.dts @@ -47,11 +47,10 @@ }; }; - gpio-keys-polled { - compatible = "gpio-keys-polled"; + gpio-keys { + compatible = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; - poll-interval = <100>; button@0 { label = "jumpstart"; diff --git a/arch/mips/boot/dts/qca/ar9331_omega.dts b/arch/mips/boot/dts/qca/ar9331_omega.dts index f70f79c4d0d5..11778abacf66 100644 --- a/arch/mips/boot/dts/qca/ar9331_omega.dts +++ b/arch/mips/boot/dts/qca/ar9331_omega.dts @@ -29,11 +29,10 @@ }; }; - gpio-keys-polled { - compatible = "gpio-keys-polled"; + gpio-keys { + compatible = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; - poll-interval = <100>; button@0 { label = "reset"; diff --git a/arch/mips/boot/dts/qca/ar9331_tl_mr3020.dts b/arch/mips/boot/dts/qca/ar9331_tl_mr3020.dts index 748131aea22e..c8290d36cfbe 100644 --- a/arch/mips/boot/dts/qca/ar9331_tl_mr3020.dts +++ b/arch/mips/boot/dts/qca/ar9331_tl_mr3020.dts @@ -47,11 +47,10 @@ }; }; - gpio-keys-polled { - compatible = "gpio-keys-polled"; + gpio-keys { + compatible = "gpio-keys"; #address-cells = <1>; #size-cells = <0>; - poll-interval = <100>; button@0 { label = "wps"; diff --git a/arch/mips/boot/ecoff.h b/arch/mips/boot/ecoff.h index b3e73c22c345..5be79ebfc3f8 100644 --- a/arch/mips/boot/ecoff.h +++ b/arch/mips/boot/ecoff.h @@ -2,14 +2,17 @@ /* * Some ECOFF definitions. */ + +#include <stdint.h> + typedef struct filehdr { - unsigned short f_magic; /* magic number */ - unsigned short f_nscns; /* number of sections */ - long f_timdat; /* time & date stamp */ - long f_symptr; /* file pointer to symbolic header */ - long f_nsyms; /* sizeof(symbolic hdr) */ - unsigned short f_opthdr; /* sizeof(optional hdr) */ - unsigned short f_flags; /* flags */ + uint16_t f_magic; /* magic number */ + uint16_t f_nscns; /* number of sections */ + int32_t f_timdat; /* time & date stamp */ + int32_t f_symptr; /* file pointer to symbolic header */ + int32_t f_nsyms; /* sizeof(symbolic hdr) */ + uint16_t f_opthdr; /* sizeof(optional hdr) */ + uint16_t f_flags; /* flags */ } FILHDR; #define FILHSZ sizeof(FILHDR) @@ -18,32 +21,32 @@ typedef struct filehdr { typedef struct scnhdr { char s_name[8]; /* section name */ - long s_paddr; /* physical address, aliased s_nlib */ - long s_vaddr; /* virtual address */ - long s_size; /* section size */ - long s_scnptr; /* file ptr to raw data for section */ - long s_relptr; /* file ptr to relocation */ - long s_lnnoptr; /* file ptr to gp histogram */ - unsigned short s_nreloc; /* number of relocation entries */ - unsigned short s_nlnno; /* number of gp histogram entries */ - long s_flags; /* flags */ + int32_t s_paddr; /* physical address, aliased s_nlib */ + int32_t s_vaddr; /* virtual address */ + int32_t s_size; /* section size */ + int32_t s_scnptr; /* file ptr to raw data for section */ + int32_t s_relptr; /* file ptr to relocation */ + int32_t s_lnnoptr; /* file ptr to gp histogram */ + uint16_t s_nreloc; /* number of relocation entries */ + uint16_t s_nlnno; /* number of gp histogram entries */ + int32_t s_flags; /* flags */ } SCNHDR; #define SCNHSZ sizeof(SCNHDR) -#define SCNROUND ((long)16) +#define SCNROUND ((int32_t)16) typedef struct aouthdr { - short magic; /* see above */ - short vstamp; /* version stamp */ - long tsize; /* text size in bytes, padded to DW bdry*/ - long dsize; /* initialized data " " */ - long bsize; /* uninitialized data " " */ - long entry; /* entry pt. */ - long text_start; /* base of text used for this file */ - long data_start; /* base of data used for this file */ - long bss_start; /* base of bss used for this file */ - long gprmask; /* general purpose register mask */ - long cprmask[4]; /* co-processor register masks */ - long gp_value; /* the gp value used for this object */ + int16_t magic; /* see above */ + int16_t vstamp; /* version stamp */ + int32_t tsize; /* text size in bytes, padded to DW bdry*/ + int32_t dsize; /* initialized data " " */ + int32_t bsize; /* uninitialized data " " */ + int32_t entry; /* entry pt. */ + int32_t text_start; /* base of text used for this file */ + int32_t data_start; /* base of data used for this file */ + int32_t bss_start; /* base of bss used for this file */ + int32_t gprmask; /* general purpose register mask */ + int32_t cprmask[4]; /* co-processor register masks */ + int32_t gp_value; /* the gp value used for this object */ } AOUTHDR; #define AOUTHSZ sizeof(AOUTHDR) diff --git a/arch/mips/boot/elf2ecoff.c b/arch/mips/boot/elf2ecoff.c index 266c8137e859..6972b97235da 100644 --- a/arch/mips/boot/elf2ecoff.c +++ b/arch/mips/boot/elf2ecoff.c @@ -43,6 +43,8 @@ #include <limits.h> #include <netinet/in.h> #include <stdlib.h> +#include <stdint.h> +#include <inttypes.h> #include "ecoff.h" @@ -55,8 +57,8 @@ /* -------------------------------------------------------------------- */ struct sect { - unsigned long vaddr; - unsigned long len; + uint32_t vaddr; + uint32_t len; }; int *symTypeTable; @@ -153,16 +155,16 @@ static char *saveRead(int file, off_t offset, off_t len, char *name) } #define swab16(x) \ - ((unsigned short)( \ - (((unsigned short)(x) & (unsigned short)0x00ffU) << 8) | \ - (((unsigned short)(x) & (unsigned short)0xff00U) >> 8) )) + ((uint16_t)( \ + (((uint16_t)(x) & (uint16_t)0x00ffU) << 8) | \ + (((uint16_t)(x) & (uint16_t)0xff00U) >> 8) )) #define swab32(x) \ ((unsigned int)( \ - (((unsigned int)(x) & (unsigned int)0x000000ffUL) << 24) | \ - (((unsigned int)(x) & (unsigned int)0x0000ff00UL) << 8) | \ - (((unsigned int)(x) & (unsigned int)0x00ff0000UL) >> 8) | \ - (((unsigned int)(x) & (unsigned int)0xff000000UL) >> 24) )) + (((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) | \ + (((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) | \ + (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) | \ + (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24) )) static void convert_elf_hdr(Elf32_Ehdr * e) { @@ -274,7 +276,7 @@ int main(int argc, char *argv[]) struct aouthdr eah; struct scnhdr esecs[6]; int infile, outfile; - unsigned long cur_vma = ULONG_MAX; + uint32_t cur_vma = UINT32_MAX; int addflag = 0; int nosecs; @@ -518,7 +520,7 @@ int main(int argc, char *argv[]) for (i = 0; i < nosecs; i++) { printf - ("Section %d: %s phys %lx size %lx file offset %lx\n", + ("Section %d: %s phys %"PRIx32" size %"PRIx32"\t file offset %"PRIx32"\n", i, esecs[i].s_name, esecs[i].s_paddr, esecs[i].s_size, esecs[i].s_scnptr); } @@ -564,17 +566,16 @@ int main(int argc, char *argv[]) the section can be loaded before copying. */ if (ph[i].p_type == PT_LOAD && ph[i].p_filesz) { if (cur_vma != ph[i].p_vaddr) { - unsigned long gap = - ph[i].p_vaddr - cur_vma; + uint32_t gap = ph[i].p_vaddr - cur_vma; char obuf[1024]; if (gap > 65536) { fprintf(stderr, - "Intersegment gap (%ld bytes) too large.\n", + "Intersegment gap (%"PRId32" bytes) too large.\n", gap); exit(1); } fprintf(stderr, - "Warning: %ld byte intersegment gap.\n", + "Warning: %d byte intersegment gap.\n", gap); memset(obuf, 0, sizeof obuf); while (gap) { diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c index 7b335ab21697..236833be6fbe 100644 --- a/arch/mips/cavium-octeon/dma-octeon.c +++ b/arch/mips/cavium-octeon/dma-octeon.c @@ -11,9 +11,7 @@ * Copyright (C) 2010 Cavium Networks, Inc. */ #include <linux/dma-direct.h> -#include <linux/scatterlist.h> #include <linux/bootmem.h> -#include <linux/export.h> #include <linux/swiotlb.h> #include <linux/types.h> #include <linux/init.h> @@ -24,10 +22,16 @@ #include <asm/octeon/octeon.h> #ifdef CONFIG_PCI +#include <linux/pci.h> #include <asm/octeon/pci-octeon.h> #include <asm/octeon/cvmx-npi-defs.h> #include <asm/octeon/cvmx-pci-defs.h> +struct octeon_dma_map_ops { + dma_addr_t (*phys_to_dma)(struct device *dev, phys_addr_t paddr); + phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr); +}; + static dma_addr_t octeon_hole_phys_to_dma(phys_addr_t paddr) { if (paddr >= CVMX_PCIE_BAR1_PHYS_BASE && paddr < (CVMX_PCIE_BAR1_PHYS_BASE + CVMX_PCIE_BAR1_PHYS_SIZE)) @@ -61,6 +65,11 @@ static phys_addr_t octeon_gen1_dma_to_phys(struct device *dev, dma_addr_t daddr) return daddr; } +static const struct octeon_dma_map_ops octeon_gen1_ops = { + .phys_to_dma = octeon_gen1_phys_to_dma, + .dma_to_phys = octeon_gen1_dma_to_phys, +}; + static dma_addr_t octeon_gen2_phys_to_dma(struct device *dev, phys_addr_t paddr) { return octeon_hole_phys_to_dma(paddr); @@ -71,6 +80,11 @@ static phys_addr_t octeon_gen2_dma_to_phys(struct device *dev, dma_addr_t daddr) return octeon_hole_dma_to_phys(daddr); } +static const struct octeon_dma_map_ops octeon_gen2_ops = { + .phys_to_dma = octeon_gen2_phys_to_dma, + .dma_to_phys = octeon_gen2_dma_to_phys, +}; + static dma_addr_t octeon_big_phys_to_dma(struct device *dev, phys_addr_t paddr) { if (paddr >= 0x410000000ull && paddr < 0x420000000ull) @@ -93,6 +107,11 @@ static phys_addr_t octeon_big_dma_to_phys(struct device *dev, dma_addr_t daddr) return daddr; } +static const struct octeon_dma_map_ops octeon_big_ops = { + .phys_to_dma = octeon_big_phys_to_dma, + .dma_to_phys = octeon_big_dma_to_phys, +}; + static dma_addr_t octeon_small_phys_to_dma(struct device *dev, phys_addr_t paddr) { @@ -121,105 +140,51 @@ static phys_addr_t octeon_small_dma_to_phys(struct device *dev, return daddr; } -#endif /* CONFIG_PCI */ - -static dma_addr_t octeon_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction direction, - unsigned long attrs) -{ - dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size, - direction, attrs); - mb(); - - return daddr; -} - -static int octeon_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction direction, unsigned long attrs) -{ - int r = swiotlb_map_sg_attrs(dev, sg, nents, direction, attrs); - mb(); - return r; -} - -static void octeon_dma_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) -{ - swiotlb_sync_single_for_device(dev, dma_handle, size, direction); - mb(); -} - -static void octeon_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sg, int nelems, enum dma_data_direction direction) -{ - swiotlb_sync_sg_for_device(dev, sg, nelems, direction); - mb(); -} - -static void *octeon_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) -{ - void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs); - - mb(); +static const struct octeon_dma_map_ops octeon_small_ops = { + .phys_to_dma = octeon_small_phys_to_dma, + .dma_to_phys = octeon_small_dma_to_phys, +}; - return ret; -} +static const struct octeon_dma_map_ops *octeon_pci_dma_ops; -static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - return paddr; -} - -static phys_addr_t octeon_unity_dma_to_phys(struct device *dev, dma_addr_t daddr) +void __init octeon_pci_dma_init(void) { - return daddr; + switch (octeon_dma_bar_type) { + case OCTEON_DMA_BAR_TYPE_PCIE: + octeon_pci_dma_ops = &octeon_gen1_ops; + break; + case OCTEON_DMA_BAR_TYPE_PCIE2: + octeon_pci_dma_ops = &octeon_gen2_ops; + break; + case OCTEON_DMA_BAR_TYPE_BIG: + octeon_pci_dma_ops = &octeon_big_ops; + break; + case OCTEON_DMA_BAR_TYPE_SMALL: + octeon_pci_dma_ops = &octeon_small_ops; + break; + default: + BUG(); + } } - -struct octeon_dma_map_ops { - const struct dma_map_ops dma_map_ops; - dma_addr_t (*phys_to_dma)(struct device *dev, phys_addr_t paddr); - phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr); -}; +#endif /* CONFIG_PCI */ dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) { - struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), - struct octeon_dma_map_ops, - dma_map_ops); - - return ops->phys_to_dma(dev, paddr); +#ifdef CONFIG_PCI + if (dev && dev_is_pci(dev)) + return octeon_pci_dma_ops->phys_to_dma(dev, paddr); +#endif + return paddr; } -EXPORT_SYMBOL(__phys_to_dma); phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) { - struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), - struct octeon_dma_map_ops, - dma_map_ops); - - return ops->dma_to_phys(dev, daddr); +#ifdef CONFIG_PCI + if (dev && dev_is_pci(dev)) + return octeon_pci_dma_ops->dma_to_phys(dev, daddr); +#endif + return daddr; } -EXPORT_SYMBOL(__dma_to_phys); - -static struct octeon_dma_map_ops octeon_linear_dma_map_ops = { - .dma_map_ops = { - .alloc = octeon_dma_alloc_coherent, - .free = swiotlb_free, - .map_page = octeon_dma_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = octeon_dma_map_sg, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = octeon_dma_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = octeon_dma_sync_sg_for_device, - .mapping_error = swiotlb_dma_mapping_error, - .dma_supported = swiotlb_dma_supported - }, - .phys_to_dma = octeon_unity_phys_to_dma, - .dma_to_phys = octeon_unity_dma_to_phys -}; char *octeon_swiotlb; @@ -283,52 +248,4 @@ void __init plat_swiotlb_setup(void) if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs, 1) == -ENOMEM) panic("Cannot allocate SWIOTLB buffer"); - - mips_dma_map_ops = &octeon_linear_dma_map_ops.dma_map_ops; } - -#ifdef CONFIG_PCI -static struct octeon_dma_map_ops _octeon_pci_dma_map_ops = { - .dma_map_ops = { - .alloc = octeon_dma_alloc_coherent, - .free = swiotlb_free, - .map_page = octeon_dma_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = octeon_dma_map_sg, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = octeon_dma_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = octeon_dma_sync_sg_for_device, - .mapping_error = swiotlb_dma_mapping_error, - .dma_supported = swiotlb_dma_supported - }, -}; - -const struct dma_map_ops *octeon_pci_dma_map_ops; - -void __init octeon_pci_dma_init(void) -{ - switch (octeon_dma_bar_type) { - case OCTEON_DMA_BAR_TYPE_PCIE2: - _octeon_pci_dma_map_ops.phys_to_dma = octeon_gen2_phys_to_dma; - _octeon_pci_dma_map_ops.dma_to_phys = octeon_gen2_dma_to_phys; - break; - case OCTEON_DMA_BAR_TYPE_PCIE: - _octeon_pci_dma_map_ops.phys_to_dma = octeon_gen1_phys_to_dma; - _octeon_pci_dma_map_ops.dma_to_phys = octeon_gen1_dma_to_phys; - break; - case OCTEON_DMA_BAR_TYPE_BIG: - _octeon_pci_dma_map_ops.phys_to_dma = octeon_big_phys_to_dma; - _octeon_pci_dma_map_ops.dma_to_phys = octeon_big_dma_to_phys; - break; - case OCTEON_DMA_BAR_TYPE_SMALL: - _octeon_pci_dma_map_ops.phys_to_dma = octeon_small_phys_to_dma; - _octeon_pci_dma_map_ops.dma_to_phys = octeon_small_dma_to_phys; - break; - default: - BUG(); - } - octeon_pci_dma_map_ops = &_octeon_pci_dma_map_ops.dma_map_ops; -} -#endif /* CONFIG_PCI */ diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c b/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c index d18ed5af62f4..b8898e2b8a6f 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-rgmii.c @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2008 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -42,9 +42,6 @@ #include <asm/octeon/cvmx-asxx-defs.h> #include <asm/octeon/cvmx-dbg-defs.h> -void __cvmx_interrupt_gmxx_enable(int interface); -void __cvmx_interrupt_asxx_enable(int block); - /** * Probe RGMII ports and determine the number present * diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c b/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c index 578283350776..a176358c5a21 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-sgmii.c @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2008 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -39,10 +39,7 @@ #include <asm/octeon/cvmx-gmxx-defs.h> #include <asm/octeon/cvmx-pcsx-defs.h> - -void __cvmx_interrupt_gmxx_enable(int interface); -void __cvmx_interrupt_pcsx_intx_en_reg_enable(int index, int block); -void __cvmx_interrupt_pcsxx_int_en_reg_enable(int index); +#include <asm/octeon/cvmx-pcsxx-defs.h> /** * Perform initialization required only once for an SGMII port. diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c b/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c index ef16aa00167b..2a574d272671 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-spi.c @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2008 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -25,10 +25,6 @@ * Contact Cavium Networks for more information ***********************license end**************************************/ -void __cvmx_interrupt_gmxx_enable(int interface); -void __cvmx_interrupt_spxx_int_msk_enable(int index); -void __cvmx_interrupt_stxx_int_msk_enable(int index); - /* * Functions for SPI initialization, configuration, * and monitoring. @@ -41,6 +37,8 @@ void __cvmx_interrupt_stxx_int_msk_enable(int index); #include <asm/octeon/cvmx-pip-defs.h> #include <asm/octeon/cvmx-pko-defs.h> +#include <asm/octeon/cvmx-spxx-defs.h> +#include <asm/octeon/cvmx-stxx-defs.h> /* * CVMX_HELPER_SPI_TIMEOUT is used to determine how long the SPI diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c index 19d54e02c185..2bb6912a580d 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2008 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -39,12 +39,9 @@ #include <asm/octeon/cvmx-pko-defs.h> #include <asm/octeon/cvmx-gmxx-defs.h> +#include <asm/octeon/cvmx-pcsx-defs.h> #include <asm/octeon/cvmx-pcsxx-defs.h> -void __cvmx_interrupt_gmxx_enable(int interface); -void __cvmx_interrupt_pcsx_intx_en_reg_enable(int index, int block); -void __cvmx_interrupt_pcsxx_int_en_reg_enable(int index); - int __cvmx_helper_xaui_enumerate(int interface) { union cvmx_gmxx_hg2_control gmx_hg2_control; diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index b3aec101a65d..8272d8c648ca 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -814,7 +814,7 @@ static int octeon_irq_ciu_set_affinity(struct irq_data *data, pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu); if (cpumask_test_cpu(cpu, dest) && enable_one) { - enable_one = 0; + enable_one = false; __set_bit(cd->bit, pen); } else { __clear_bit(cd->bit, pen); diff --git a/arch/mips/cavium-octeon/octeon-platform.c b/arch/mips/cavium-octeon/octeon-platform.c index 8505db478904..807cadaf554e 100644 --- a/arch/mips/cavium-octeon/octeon-platform.c +++ b/arch/mips/cavium-octeon/octeon-platform.c @@ -322,6 +322,7 @@ static int __init octeon_ehci_device_init(void) return 0; pd = of_find_device_by_node(ehci_node); + of_node_put(ehci_node); if (!pd) return 0; @@ -384,6 +385,7 @@ static int __init octeon_ohci_device_init(void) return 0; pd = of_find_device_by_node(ohci_node); + of_node_put(ohci_node); if (!pd) return 0; @@ -1067,6 +1069,6 @@ end_led: static int __init octeon_publish_devices(void) { - return of_platform_bus_probe(NULL, octeon_ids, NULL); + return of_platform_populate(NULL, octeon_ids, NULL, NULL); } arch_initcall(octeon_publish_devices); diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index a8034d0dcade..c2426232db06 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -36,6 +36,7 @@ #include <asm/mipsregs.h> #include <asm/bootinfo.h> #include <asm/sections.h> +#include <asm/setup.h> #include <asm/time.h> #include <asm/octeon/octeon.h> @@ -1108,7 +1109,7 @@ void __init plat_mem_setup(void) * Emit one character to the boot UART. Exported for use by the * watchdog timer. */ -int prom_putchar(char c) +void prom_putchar(char c) { uint64_t lsrval; @@ -1119,7 +1120,6 @@ int prom_putchar(char c) /* Write the byte */ cvmx_write_csr(CVMX_MIO_UARTX_THR(octeon_uart), c & 0xffull); - return 1; } EXPORT_SYMBOL(prom_putchar); @@ -1154,11 +1154,7 @@ void __init prom_free_prom_memory(void) } void __init octeon_fill_mac_addresses(void); -int octeon_prune_device_tree(void); -extern const char __appended_dtb; -extern const char __dtb_octeon_3xxx_begin; -extern const char __dtb_octeon_68xx_begin; void __init device_tree_init(void) { const void *fdt; diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index be23fd25eeaa..030ff9c205fb 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -92,6 +92,8 @@ CONFIG_SERIAL_OF_PLATFORM=y # CONFIG_HW_RANDOM is not set CONFIG_I2C=y CONFIG_I2C_JZ4780=y +CONFIG_SPI=y +CONFIG_SPI_GPIO=y CONFIG_GPIO_SYSFS=y CONFIG_GPIO_INGENIC=y # CONFIG_HWMON is not set diff --git a/arch/mips/configs/generic_defconfig b/arch/mips/configs/generic_defconfig index 26b1cd5ffbf5..684c9dcba126 100644 --- a/arch/mips/configs/generic_defconfig +++ b/arch/mips/configs/generic_defconfig @@ -43,9 +43,6 @@ CONFIG_NETFILTER=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_SCSI=y -# CONFIG_INPUT_MOUSEDEV is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set # CONFIG_SERIO is not set CONFIG_HW_RANDOM=y # CONFIG_HWMON is not set diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index df8a9a15ca83..81058295d35f 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -317,6 +317,7 @@ CONFIG_MOUSE_PS2_ELANTECH=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index 14df9ef15d40..5c10cddc39d3 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -328,6 +328,7 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/malta_kvm_guest_defconfig b/arch/mips/configs/malta_kvm_guest_defconfig index 25092e344574..bb694f5065f1 100644 --- a/arch/mips/configs/malta_kvm_guest_defconfig +++ b/arch/mips/configs/malta_kvm_guest_defconfig @@ -330,6 +330,7 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig index 210bf609f785..5b5306b80576 100644 --- a/arch/mips/configs/malta_qemu_32r6_defconfig +++ b/arch/mips/configs/malta_qemu_32r6_defconfig @@ -133,6 +133,7 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig index e5934aa98397..85543599448f 100644 --- a/arch/mips/configs/maltaaprp_defconfig +++ b/arch/mips/configs/maltaaprp_defconfig @@ -133,6 +133,7 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig index cb2ca11c1789..067bb84ac916 100644 --- a/arch/mips/configs/maltasmvp_defconfig +++ b/arch/mips/configs/maltasmvp_defconfig @@ -134,6 +134,7 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig index be29fcec69fc..dfc78c3172a3 100644 --- a/arch/mips/configs/maltasmvp_eva_defconfig +++ b/arch/mips/configs/maltasmvp_eva_defconfig @@ -137,6 +137,7 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig index 40462d4c90a0..50a2288c69f8 100644 --- a/arch/mips/configs/maltaup_defconfig +++ b/arch/mips/configs/maltaup_defconfig @@ -132,6 +132,7 @@ CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index 4e50176cb3df..99a19cf5f9ba 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -326,6 +326,7 @@ CONFIG_MOUSE_PS2_ELANTECH=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_PIIX4_POWEROFF=y CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_FB=y diff --git a/arch/mips/fw/arc/arc_con.c b/arch/mips/fw/arc/arc_con.c index 769d4b9ac82e..365e3913231e 100644 --- a/arch/mips/fw/arc/arc_con.c +++ b/arch/mips/fw/arc/arc_con.c @@ -12,6 +12,7 @@ #include <linux/init.h> #include <linux/console.h> #include <linux/fs.h> +#include <asm/setup.h> #include <asm/sgialib.h> static void prom_console_write(struct console *co, const char *s, diff --git a/arch/mips/fw/arc/promlib.c b/arch/mips/fw/arc/promlib.c index 7e8ba5ce95be..be381307fbb0 100644 --- a/arch/mips/fw/arc/promlib.c +++ b/arch/mips/fw/arc/promlib.c @@ -9,6 +9,7 @@ #include <linux/kernel.h> #include <asm/sgialib.h> #include <asm/bcache.h> +#include <asm/setup.h> /* * IP22 boardcache is not compatible with board caches. Thus we disable it diff --git a/arch/mips/fw/sni/sniprom.c b/arch/mips/fw/sni/sniprom.c index 6aa264b9856a..8772617b64ce 100644 --- a/arch/mips/fw/sni/sniprom.c +++ b/arch/mips/fw/sni/sniprom.c @@ -19,6 +19,7 @@ #include <asm/mipsprom.h> #include <asm/mipsregs.h> #include <asm/bootinfo.h> +#include <asm/setup.h> /* special SNI prom calls */ /* diff --git a/arch/mips/generic/Kconfig b/arch/mips/generic/Kconfig index ba9b2c8cce68..08e33c6b2539 100644 --- a/arch/mips/generic/Kconfig +++ b/arch/mips/generic/Kconfig @@ -35,13 +35,13 @@ config LEGACY_BOARD_OCELOT depends on LEGACY_BOARD_SEAD3=n select LEGACY_BOARDS select MSCC_OCELOT + select SYS_HAS_EARLY_PRINTK + select USE_GENERIC_EARLY_PRINTK_8250 config MSCC_OCELOT bool select GPIOLIB select MSCC_OCELOT_IRQ - select SYS_HAS_EARLY_PRINTK - select USE_GENERIC_EARLY_PRINTK_8250 comment "FIT/UHI Boards" @@ -65,6 +65,14 @@ config FIT_IMAGE_FDT_XILFPGA Enable this to include the FDT for the MIPSfpga platform from Imagination Technologies in the FIT kernel image. +config FIT_IMAGE_FDT_OCELOT_PCB123 + bool "Include FDT for Microsemi Ocelot PCB123" + select MSCC_OCELOT + help + Enable this to include the FDT for the Ocelot PCB123 platform + from Microsemi in the FIT kernel image. + This requires u-boot on the platform. + config VIRT_BOARD_RANCHU bool "Support Ranchu platform for Android emulator" help diff --git a/arch/mips/generic/Platform b/arch/mips/generic/Platform index 0dd0d5d460a5..879cb80396c8 100644 --- a/arch/mips/generic/Platform +++ b/arch/mips/generic/Platform @@ -16,4 +16,5 @@ all-$(CONFIG_MIPS_GENERIC) := vmlinux.gz.itb its-y := vmlinux.its.S its-$(CONFIG_FIT_IMAGE_FDT_BOSTON) += board-boston.its.S its-$(CONFIG_FIT_IMAGE_FDT_NI169445) += board-ni169445.its.S +its-$(CONFIG_FIT_IMAGE_FDT_OCELOT_PCB123) += board-ocelot_pcb123.its.S its-$(CONFIG_FIT_IMAGE_FDT_XILFPGA) += board-xilfpga.its.S diff --git a/arch/mips/generic/board-ocelot_pcb123.its.S b/arch/mips/generic/board-ocelot_pcb123.its.S new file mode 100644 index 000000000000..5a7d5e1c878a --- /dev/null +++ b/arch/mips/generic/board-ocelot_pcb123.its.S @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/ { + images { + fdt@ocelot_pcb123 { + description = "MSCC Ocelot PCB123 Device Tree"; + data = /incbin/("boot/dts/mscc/ocelot_pcb123.dtb"); + type = "flat_dt"; + arch = "mips"; + compression = "none"; + hash@0 { + algo = "sha1"; + }; + }; + }; + + configurations { + conf@ocelot_pcb123 { + description = "Ocelot Linux kernel"; + kernel = "kernel@0"; + fdt = "fdt@ocelot_pcb123"; + }; + }; +}; diff --git a/arch/mips/generic/init.c b/arch/mips/generic/init.c index 5ba6fcc26fa7..a106f8113842 100644 --- a/arch/mips/generic/init.c +++ b/arch/mips/generic/init.c @@ -14,7 +14,6 @@ #include <linux/init.h> #include <linux/irqchip.h> #include <linux/of_fdt.h> -#include <linux/of_platform.h> #include <asm/bootinfo.h> #include <asm/fw/fw.h> @@ -204,22 +203,11 @@ void __init arch_init_irq(void) "mti,cpu-interrupt-controller"); if (!cpu_has_veic && !intc_node) mips_cpu_irq_init(); + of_node_put(intc_node); irqchip_init(); } -static int __init publish_devices(void) -{ - if (!of_have_populated_dt()) - panic("Device-tree not present"); - - if (of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL)) - panic("Failed to populate DT"); - - return 0; -} -arch_initcall(publish_devices); - void __init prom_free_prom_memory(void) { } diff --git a/arch/mips/generic/yamon-dt.c b/arch/mips/generic/yamon-dt.c index b408dac722ac..7ba4ad5cc1d6 100644 --- a/arch/mips/generic/yamon-dt.c +++ b/arch/mips/generic/yamon-dt.c @@ -28,8 +28,6 @@ __init int yamon_dt_append_cmdline(void *fdt) /* find or add chosen node */ chosen_off = fdt_path_offset(fdt, "/chosen"); if (chosen_off == -FDT_ERR_NOTFOUND) - chosen_off = fdt_path_offset(fdt, "/chosen@0"); - if (chosen_off == -FDT_ERR_NOTFOUND) chosen_off = fdt_add_subnode(fdt, 0, "chosen"); if (chosen_off < 0) { pr_err("Unable to find or add DT chosen node: %d\n", @@ -221,8 +219,6 @@ __init int yamon_dt_serial_config(void *fdt) /* find or add chosen node */ chosen_off = fdt_path_offset(fdt, "/chosen"); if (chosen_off == -FDT_ERR_NOTFOUND) - chosen_off = fdt_path_offset(fdt, "/chosen@0"); - if (chosen_off == -FDT_ERR_NOTFOUND) chosen_off = fdt_add_subnode(fdt, 0, "chosen"); if (chosen_off < 0) { pr_err("Unable to find or add DT chosen node: %d\n", diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 45d541baf359..58351e48421e 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild @@ -8,6 +8,7 @@ generic-y += irq_work.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h +generic-y += msi.h generic-y += parport.h generic-y += percpu.h generic-y += preempt.h diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 0ab176bdb8e8..0269b3de8b51 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -22,6 +22,17 @@ #include <asm/cmpxchg.h> #include <asm/war.h> +/* + * Using a branch-likely instruction to check the result of an sc instruction + * works around a bug present in R10000 CPUs prior to revision 3.0 that could + * cause ll-sc sequences to execute non-atomically. + */ +#if R10000_LLSC_WAR +# define __scbeqz "beqzl" +#else +# define __scbeqz "beqz" +#endif + #define ATOMIC_INIT(i) { (i) } /* @@ -44,31 +55,18 @@ #define ATOMIC_OP(op, c_op, asm_op) \ static __inline__ void atomic_##op(int i, atomic_t * v) \ { \ - if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + if (kernel_uses_llsc) { \ int temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: ll %0, %1 # atomic_" #op " \n" \ " " #asm_op " %0, %2 \n" \ " sc %0, %1 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " .set mips0 \n" \ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - int temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " ll %0, %1 # atomic_" #op "\n" \ - " " #asm_op " %0, %2 \n" \ - " sc %0, %1 \n" \ - " .set mips0 \n" \ - : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i)); \ - } while (unlikely(!temp)); \ } else { \ unsigned long flags; \ \ @@ -83,36 +81,20 @@ static __inline__ int atomic_##op##_return_relaxed(int i, atomic_t * v) \ { \ int result; \ \ - if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + if (kernel_uses_llsc) { \ int temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: ll %1, %2 # atomic_" #op "_return \n" \ " " #asm_op " %0, %1, %3 \n" \ " sc %0, %2 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " " #asm_op " %0, %1, %3 \n" \ " .set mips0 \n" \ : "=&r" (result), "=&r" (temp), \ "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - int temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " ll %1, %2 # atomic_" #op "_return \n" \ - " " #asm_op " %0, %1, %3 \n" \ - " sc %0, %2 \n" \ - " .set mips0 \n" \ - : "=&r" (result), "=&r" (temp), \ - "+" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i)); \ - } while (unlikely(!result)); \ - \ - result = temp; result c_op i; \ } else { \ unsigned long flags; \ \ @@ -131,36 +113,20 @@ static __inline__ int atomic_fetch_##op##_relaxed(int i, atomic_t * v) \ { \ int result; \ \ - if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + if (kernel_uses_llsc) { \ int temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: ll %1, %2 # atomic_fetch_" #op " \n" \ " " #asm_op " %0, %1, %3 \n" \ " sc %0, %2 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " move %0, %1 \n" \ " .set mips0 \n" \ : "=&r" (result), "=&r" (temp), \ "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - int temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " ll %1, %2 # atomic_fetch_" #op " \n" \ - " " #asm_op " %0, %1, %3 \n" \ - " sc %0, %2 \n" \ - " .set mips0 \n" \ - : "=&r" (result), "=&r" (temp), \ - "+" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i)); \ - } while (unlikely(!result)); \ - \ - result = temp; \ } else { \ unsigned long flags; \ \ @@ -218,38 +184,17 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) smp_mb__before_llsc(); - if (kernel_uses_llsc && R10000_LLSC_WAR) { - int temp; - - __asm__ __volatile__( - " .set arch=r4000 \n" - "1: ll %1, %2 # atomic_sub_if_positive\n" - " subu %0, %1, %3 \n" - " bltz %0, 1f \n" - " sc %0, %2 \n" - " .set noreorder \n" - " beqzl %0, 1b \n" - " subu %0, %1, %3 \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), - "+" GCC_OFF_SMALL_ASM() (v->counter) - : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) - : "memory"); - } else if (kernel_uses_llsc) { + if (kernel_uses_llsc) { int temp; __asm__ __volatile__( " .set "MIPS_ISA_LEVEL" \n" "1: ll %1, %2 # atomic_sub_if_positive\n" " subu %0, %1, %3 \n" + " move %1, %0 \n" " bltz %0, 1f \n" - " sc %0, %2 \n" - " .set noreorder \n" - " beqz %0, 1b \n" - " subu %0, %1, %3 \n" - " .set reorder \n" + " sc %1, %2 \n" + "\t" __scbeqz " %1, 1b \n" "1: \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), @@ -274,97 +219,12 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) -/** - * __atomic_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) - -/* - * atomic_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) - -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -/* - * atomic_dec_and_test - decrement by 1 and test - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) - /* * atomic_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t */ #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) -/* - * atomic_inc - increment atomic variable - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1. - */ -#define atomic_inc(v) atomic_add(1, (v)) - -/* - * atomic_dec - decrement and test - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1. - */ -#define atomic_dec(v) atomic_sub(1, (v)) - -/* - * atomic_add_negative - add and test if negative - * @v: pointer of type atomic_t - * @i: integer value to add - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -#define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0) - #ifdef CONFIG_64BIT #define ATOMIC64_INIT(i) { (i) } @@ -386,31 +246,18 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) #define ATOMIC64_OP(op, c_op, asm_op) \ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ { \ - if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + if (kernel_uses_llsc) { \ long temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: lld %0, %1 # atomic64_" #op " \n" \ " " #asm_op " %0, %2 \n" \ " scd %0, %1 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " .set mips0 \n" \ : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - long temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " lld %0, %1 # atomic64_" #op "\n" \ - " " #asm_op " %0, %2 \n" \ - " scd %0, %1 \n" \ - " .set mips0 \n" \ - : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i)); \ - } while (unlikely(!temp)); \ } else { \ unsigned long flags; \ \ @@ -425,37 +272,20 @@ static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \ { \ long result; \ \ - if (kernel_uses_llsc && R10000_LLSC_WAR) { \ + if (kernel_uses_llsc) { \ long temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: lld %1, %2 # atomic64_" #op "_return\n" \ " " #asm_op " %0, %1, %3 \n" \ " scd %0, %2 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " " #asm_op " %0, %1, %3 \n" \ " .set mips0 \n" \ : "=&r" (result), "=&r" (temp), \ "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - long temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " lld %1, %2 # atomic64_" #op "_return\n" \ - " " #asm_op " %0, %1, %3 \n" \ - " scd %0, %2 \n" \ - " .set mips0 \n" \ - : "=&r" (result), "=&r" (temp), \ - "=" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \ - : "memory"); \ - } while (unlikely(!result)); \ - \ - result = temp; result c_op i; \ } else { \ unsigned long flags; \ \ @@ -478,33 +308,16 @@ static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \ long temp; \ \ __asm__ __volatile__( \ - " .set arch=r4000 \n" \ + " .set "MIPS_ISA_LEVEL" \n" \ "1: lld %1, %2 # atomic64_fetch_" #op "\n" \ " " #asm_op " %0, %1, %3 \n" \ " scd %0, %2 \n" \ - " beqzl %0, 1b \n" \ + "\t" __scbeqz " %0, 1b \n" \ " move %0, %1 \n" \ " .set mips0 \n" \ : "=&r" (result), "=&r" (temp), \ "+" GCC_OFF_SMALL_ASM() (v->counter) \ : "Ir" (i)); \ - } else if (kernel_uses_llsc) { \ - long temp; \ - \ - do { \ - __asm__ __volatile__( \ - " .set "MIPS_ISA_LEVEL" \n" \ - " lld %1, %2 # atomic64_fetch_" #op "\n" \ - " " #asm_op " %0, %1, %3 \n" \ - " scd %0, %2 \n" \ - " .set mips0 \n" \ - : "=&r" (result), "=&r" (temp), \ - "=" GCC_OFF_SMALL_ASM() (v->counter) \ - : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \ - : "memory"); \ - } while (unlikely(!result)); \ - \ - result = temp; \ } else { \ unsigned long flags; \ \ @@ -563,38 +376,17 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) smp_mb__before_llsc(); - if (kernel_uses_llsc && R10000_LLSC_WAR) { - long temp; - - __asm__ __volatile__( - " .set arch=r4000 \n" - "1: lld %1, %2 # atomic64_sub_if_positive\n" - " dsubu %0, %1, %3 \n" - " bltz %0, 1f \n" - " scd %0, %2 \n" - " .set noreorder \n" - " beqzl %0, 1b \n" - " dsubu %0, %1, %3 \n" - " .set reorder \n" - "1: \n" - " .set mips0 \n" - : "=&r" (result), "=&r" (temp), - "=" GCC_OFF_SMALL_ASM() (v->counter) - : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) - : "memory"); - } else if (kernel_uses_llsc) { + if (kernel_uses_llsc) { long temp; __asm__ __volatile__( " .set "MIPS_ISA_LEVEL" \n" "1: lld %1, %2 # atomic64_sub_if_positive\n" " dsubu %0, %1, %3 \n" + " move %1, %0 \n" " bltz %0, 1f \n" - " scd %0, %2 \n" - " .set noreorder \n" - " beqz %0, 1b \n" - " dsubu %0, %1, %3 \n" - " .set reorder \n" + " scd %1, %2 \n" + "\t" __scbeqz " %1, 1b \n" "1: \n" " .set mips0 \n" : "=&r" (result), "=&r" (temp), @@ -620,99 +412,12 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new))) -/** - * atomic64_add_unless - add unless the number is a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns true iff @v was not @u. - */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - c = atomic64_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic64_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c != (u); -} - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -#define atomic64_dec_return(v) atomic64_sub_return(1, (v)) -#define atomic64_inc_return(v) atomic64_add_return(1, (v)) - -/* - * atomic64_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic64_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) - -/* - * atomic64_inc_and_test - increment and test - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) - -/* - * atomic64_dec_and_test - decrement by 1 and test - * @v: pointer of type atomic64_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) - /* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic64_t */ #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v) -/* - * atomic64_inc - increment atomic variable - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1. - */ -#define atomic64_inc(v) atomic64_add(1, (v)) - -/* - * atomic64_dec - decrement and test - * @v: pointer of type atomic64_t - * - * Atomically decrements @v by 1. - */ -#define atomic64_dec(v) atomic64_sub(1, (v)) - -/* - * atomic64_add_negative - add and test if negative - * @v: pointer of type atomic64_t - * @i: integer value to add - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -#define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0) - #endif /* CONFIG_64BIT */ #endif /* _ASM_ATOMIC_H */ diff --git a/arch/mips/include/asm/bmips.h b/arch/mips/include/asm/bmips.h index b3e2975f83d3..bf6a8afd7ad2 100644 --- a/arch/mips/include/asm/bmips.h +++ b/arch/mips/include/asm/bmips.h @@ -123,22 +123,6 @@ static inline void bmips_write_zscm_reg(unsigned int offset, unsigned long data) barrier(); } -static inline void bmips_post_dma_flush(struct device *dev) -{ - void __iomem *cbr = BMIPS_GET_CBR(); - u32 cfg; - - if (boot_cpu_type() != CPU_BMIPS3300 && - boot_cpu_type() != CPU_BMIPS4350 && - boot_cpu_type() != CPU_BMIPS4380) - return; - - /* Flush stale data out of the readahead cache */ - cfg = __raw_readl(cbr + BMIPS_RAC_CONFIG); - __raw_writel(cfg | 0x100, cbr + BMIPS_RAC_CONFIG); - __raw_readl(cbr + BMIPS_RAC_CONFIG); -} - #endif /* !defined(__ASSEMBLY__) */ #endif /* _ASM_BMIPS_H */ diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 9cdb4e4ce258..0edba3e75747 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -14,39 +14,77 @@ #include <asm/isa-rev.h> #include <cpu-feature-overrides.h> +#define __ase(ase) (cpu_data[0].ases & (ase)) +#define __opt(opt) (cpu_data[0].options & (opt)) + +/* + * Check if MIPS_ISA_REV is >= isa *and* an option or ASE is detected during + * boot (typically by cpu_probe()). + * + * Note that these should only be used in cases where a kernel built for an + * older ISA *cannot* run on a CPU which supports the feature in question. For + * example this may be used for features introduced with MIPSr6, since a kernel + * built for an older ISA cannot run on a MIPSr6 CPU. This should not be used + * for MIPSr2 features however, since a MIPSr1 or earlier kernel might run on a + * MIPSr2 CPU. + */ +#define __isa_ge_and_ase(isa, ase) ((MIPS_ISA_REV >= (isa)) && __ase(ase)) +#define __isa_ge_and_opt(isa, opt) ((MIPS_ISA_REV >= (isa)) && __opt(opt)) + +/* + * Check if MIPS_ISA_REV is >= isa *or* an option or ASE is detected during + * boot (typically by cpu_probe()). + * + * These are for use with features that are optional up until a particular ISA + * revision & then become required. + */ +#define __isa_ge_or_ase(isa, ase) ((MIPS_ISA_REV >= (isa)) || __ase(ase)) +#define __isa_ge_or_opt(isa, opt) ((MIPS_ISA_REV >= (isa)) || __opt(opt)) + +/* + * Check if MIPS_ISA_REV is < isa *and* an option or ASE is detected during + * boot (typically by cpu_probe()). + * + * These are for use with features that are optional up until a particular ISA + * revision & are then removed - ie. no longer present in any CPU implementing + * the given ISA revision. + */ +#define __isa_lt_and_ase(isa, ase) ((MIPS_ISA_REV < (isa)) && __ase(ase)) +#define __isa_lt_and_opt(isa, opt) ((MIPS_ISA_REV < (isa)) && __opt(opt)) + /* * SMP assumption: Options of CPU 0 are a superset of all processors. * This is true for all known MIPS systems. */ #ifndef cpu_has_tlb -#define cpu_has_tlb (cpu_data[0].options & MIPS_CPU_TLB) +#define cpu_has_tlb __opt(MIPS_CPU_TLB) #endif #ifndef cpu_has_ftlb -#define cpu_has_ftlb (cpu_data[0].options & MIPS_CPU_FTLB) +#define cpu_has_ftlb __opt(MIPS_CPU_FTLB) #endif #ifndef cpu_has_tlbinv -#define cpu_has_tlbinv (cpu_data[0].options & MIPS_CPU_TLBINV) +#define cpu_has_tlbinv __opt(MIPS_CPU_TLBINV) #endif #ifndef cpu_has_segments -#define cpu_has_segments (cpu_data[0].options & MIPS_CPU_SEGMENTS) +#define cpu_has_segments __opt(MIPS_CPU_SEGMENTS) #endif #ifndef cpu_has_eva -#define cpu_has_eva (cpu_data[0].options & MIPS_CPU_EVA) +#define cpu_has_eva __opt(MIPS_CPU_EVA) #endif #ifndef cpu_has_htw -#define cpu_has_htw (cpu_data[0].options & MIPS_CPU_HTW) +#define cpu_has_htw __opt(MIPS_CPU_HTW) #endif #ifndef cpu_has_ldpte -#define cpu_has_ldpte (cpu_data[0].options & MIPS_CPU_LDPTE) +#define cpu_has_ldpte __opt(MIPS_CPU_LDPTE) #endif #ifndef cpu_has_rixiex -#define cpu_has_rixiex (cpu_data[0].options & MIPS_CPU_RIXIEX) +#define cpu_has_rixiex __isa_ge_or_opt(6, MIPS_CPU_RIXIEX) #endif #ifndef cpu_has_maar -#define cpu_has_maar (cpu_data[0].options & MIPS_CPU_MAAR) +#define cpu_has_maar __opt(MIPS_CPU_MAAR) #endif #ifndef cpu_has_rw_llb -#define cpu_has_rw_llb (cpu_data[0].options & MIPS_CPU_RW_LLB) +#define cpu_has_rw_llb __isa_ge_or_opt(6, MIPS_CPU_RW_LLB) #endif /* @@ -59,18 +97,18 @@ #define cpu_has_3kex (!cpu_has_4kex) #endif #ifndef cpu_has_4kex -#define cpu_has_4kex (cpu_data[0].options & MIPS_CPU_4KEX) +#define cpu_has_4kex __isa_ge_or_opt(1, MIPS_CPU_4KEX) #endif #ifndef cpu_has_3k_cache -#define cpu_has_3k_cache (cpu_data[0].options & MIPS_CPU_3K_CACHE) +#define cpu_has_3k_cache __isa_lt_and_opt(1, MIPS_CPU_3K_CACHE) #endif #define cpu_has_6k_cache 0 #define cpu_has_8k_cache 0 #ifndef cpu_has_4k_cache -#define cpu_has_4k_cache (cpu_data[0].options & MIPS_CPU_4K_CACHE) +#define cpu_has_4k_cache __isa_ge_or_opt(1, MIPS_CPU_4K_CACHE) #endif #ifndef cpu_has_tx39_cache -#define cpu_has_tx39_cache (cpu_data[0].options & MIPS_CPU_TX39_CACHE) +#define cpu_has_tx39_cache __opt(MIPS_CPU_TX39_CACHE) #endif #ifndef cpu_has_octeon_cache #define cpu_has_octeon_cache 0 @@ -83,92 +121,92 @@ #define raw_cpu_has_fpu cpu_has_fpu #endif #ifndef cpu_has_32fpr -#define cpu_has_32fpr (cpu_data[0].options & MIPS_CPU_32FPR) +#define cpu_has_32fpr __isa_ge_or_opt(1, MIPS_CPU_32FPR) #endif #ifndef cpu_has_counter -#define cpu_has_counter (cpu_data[0].options & MIPS_CPU_COUNTER) +#define cpu_has_counter __opt(MIPS_CPU_COUNTER) #endif #ifndef cpu_has_watch -#define cpu_has_watch (cpu_data[0].options & MIPS_CPU_WATCH) +#define cpu_has_watch __opt(MIPS_CPU_WATCH) #endif #ifndef cpu_has_divec -#define cpu_has_divec (cpu_data[0].options & MIPS_CPU_DIVEC) +#define cpu_has_divec __isa_ge_or_opt(1, MIPS_CPU_DIVEC) #endif #ifndef cpu_has_vce -#define cpu_has_vce (cpu_data[0].options & MIPS_CPU_VCE) +#define cpu_has_vce __opt(MIPS_CPU_VCE) #endif #ifndef cpu_has_cache_cdex_p -#define cpu_has_cache_cdex_p (cpu_data[0].options & MIPS_CPU_CACHE_CDEX_P) +#define cpu_has_cache_cdex_p __opt(MIPS_CPU_CACHE_CDEX_P) #endif #ifndef cpu_has_cache_cdex_s -#define cpu_has_cache_cdex_s (cpu_data[0].options & MIPS_CPU_CACHE_CDEX_S) +#define cpu_has_cache_cdex_s __opt(MIPS_CPU_CACHE_CDEX_S) #endif #ifndef cpu_has_prefetch -#define cpu_has_prefetch (cpu_data[0].options & MIPS_CPU_PREFETCH) +#define cpu_has_prefetch __isa_ge_or_opt(1, MIPS_CPU_PREFETCH) #endif #ifndef cpu_has_mcheck -#define cpu_has_mcheck (cpu_data[0].options & MIPS_CPU_MCHECK) +#define cpu_has_mcheck __isa_ge_or_opt(1, MIPS_CPU_MCHECK) #endif #ifndef cpu_has_ejtag -#define cpu_has_ejtag (cpu_data[0].options & MIPS_CPU_EJTAG) +#define cpu_has_ejtag __opt(MIPS_CPU_EJTAG) #endif #ifndef cpu_has_llsc -#define cpu_has_llsc (cpu_data[0].options & MIPS_CPU_LLSC) +#define cpu_has_llsc __isa_ge_or_opt(1, MIPS_CPU_LLSC) #endif #ifndef cpu_has_bp_ghist -#define cpu_has_bp_ghist (cpu_data[0].options & MIPS_CPU_BP_GHIST) +#define cpu_has_bp_ghist __opt(MIPS_CPU_BP_GHIST) #endif #ifndef kernel_uses_llsc #define kernel_uses_llsc cpu_has_llsc #endif #ifndef cpu_has_guestctl0ext -#define cpu_has_guestctl0ext (cpu_data[0].options & MIPS_CPU_GUESTCTL0EXT) +#define cpu_has_guestctl0ext __opt(MIPS_CPU_GUESTCTL0EXT) #endif #ifndef cpu_has_guestctl1 -#define cpu_has_guestctl1 (cpu_data[0].options & MIPS_CPU_GUESTCTL1) +#define cpu_has_guestctl1 __opt(MIPS_CPU_GUESTCTL1) #endif #ifndef cpu_has_guestctl2 -#define cpu_has_guestctl2 (cpu_data[0].options & MIPS_CPU_GUESTCTL2) +#define cpu_has_guestctl2 __opt(MIPS_CPU_GUESTCTL2) #endif #ifndef cpu_has_guestid -#define cpu_has_guestid (cpu_data[0].options & MIPS_CPU_GUESTID) +#define cpu_has_guestid __opt(MIPS_CPU_GUESTID) #endif #ifndef cpu_has_drg -#define cpu_has_drg (cpu_data[0].options & MIPS_CPU_DRG) +#define cpu_has_drg __opt(MIPS_CPU_DRG) #endif #ifndef cpu_has_mips16 -#define cpu_has_mips16 (cpu_data[0].ases & MIPS_ASE_MIPS16) +#define cpu_has_mips16 __isa_lt_and_ase(6, MIPS_ASE_MIPS16) #endif #ifndef cpu_has_mips16e2 -#define cpu_has_mips16e2 (cpu_data[0].ases & MIPS_ASE_MIPS16E2) +#define cpu_has_mips16e2 __isa_lt_and_ase(6, MIPS_ASE_MIPS16E2) #endif #ifndef cpu_has_mdmx -#define cpu_has_mdmx (cpu_data[0].ases & MIPS_ASE_MDMX) +#define cpu_has_mdmx __isa_lt_and_ase(6, MIPS_ASE_MDMX) #endif #ifndef cpu_has_mips3d -#define cpu_has_mips3d (cpu_data[0].ases & MIPS_ASE_MIPS3D) +#define cpu_has_mips3d __isa_lt_and_ase(6, MIPS_ASE_MIPS3D) #endif #ifndef cpu_has_smartmips -#define cpu_has_smartmips (cpu_data[0].ases & MIPS_ASE_SMARTMIPS) +#define cpu_has_smartmips __isa_lt_and_ase(6, MIPS_ASE_SMARTMIPS) #endif #ifndef cpu_has_rixi -#define cpu_has_rixi (cpu_data[0].options & MIPS_CPU_RIXI) +#define cpu_has_rixi __isa_ge_or_opt(6, MIPS_CPU_RIXI) #endif #ifndef cpu_has_mmips # ifdef CONFIG_SYS_SUPPORTS_MICROMIPS -# define cpu_has_mmips (cpu_data[0].options & MIPS_CPU_MICROMIPS) +# define cpu_has_mmips __opt(MIPS_CPU_MICROMIPS) # else # define cpu_has_mmips 0 # endif #endif #ifndef cpu_has_lpa -#define cpu_has_lpa (cpu_data[0].options & MIPS_CPU_LPA) +#define cpu_has_lpa __opt(MIPS_CPU_LPA) #endif #ifndef cpu_has_mvh -#define cpu_has_mvh (cpu_data[0].options & MIPS_CPU_MVH) +#define cpu_has_mvh __opt(MIPS_CPU_MVH) #endif #ifndef cpu_has_xpa #define cpu_has_xpa (cpu_has_lpa && cpu_has_mvh) @@ -338,32 +376,32 @@ #endif #ifndef cpu_has_dsp -#define cpu_has_dsp (cpu_data[0].ases & MIPS_ASE_DSP) +#define cpu_has_dsp __ase(MIPS_ASE_DSP) #endif #ifndef cpu_has_dsp2 -#define cpu_has_dsp2 (cpu_data[0].ases & MIPS_ASE_DSP2P) +#define cpu_has_dsp2 __ase(MIPS_ASE_DSP2P) #endif #ifndef cpu_has_dsp3 -#define cpu_has_dsp3 (cpu_data[0].ases & MIPS_ASE_DSP3) +#define cpu_has_dsp3 __ase(MIPS_ASE_DSP3) #endif #ifndef cpu_has_mipsmt -#define cpu_has_mipsmt (cpu_data[0].ases & MIPS_ASE_MIPSMT) +#define cpu_has_mipsmt __isa_lt_and_ase(6, MIPS_ASE_MIPSMT) #endif #ifndef cpu_has_vp -#define cpu_has_vp (cpu_data[0].options & MIPS_CPU_VP) +#define cpu_has_vp __isa_ge_and_opt(6, MIPS_CPU_VP) #endif #ifndef cpu_has_userlocal -#define cpu_has_userlocal (cpu_data[0].options & MIPS_CPU_ULRI) +#define cpu_has_userlocal __isa_ge_or_opt(6, MIPS_CPU_ULRI) #endif #ifdef CONFIG_32BIT # ifndef cpu_has_nofpuex -# define cpu_has_nofpuex (cpu_data[0].options & MIPS_CPU_NOFPUEX) +# define cpu_has_nofpuex __isa_lt_and_opt(1, MIPS_CPU_NOFPUEX) # endif # ifndef cpu_has_64bits # define cpu_has_64bits (cpu_data[0].isa_level & MIPS_CPU_ISA_64BIT) @@ -405,19 +443,19 @@ #endif #if defined(CONFIG_CPU_MIPSR2_IRQ_VI) && !defined(cpu_has_vint) -# define cpu_has_vint (cpu_data[0].options & MIPS_CPU_VINT) +# define cpu_has_vint __opt(MIPS_CPU_VINT) #elif !defined(cpu_has_vint) # define cpu_has_vint 0 #endif #if defined(CONFIG_CPU_MIPSR2_IRQ_EI) && !defined(cpu_has_veic) -# define cpu_has_veic (cpu_data[0].options & MIPS_CPU_VEIC) +# define cpu_has_veic __opt(MIPS_CPU_VEIC) #elif !defined(cpu_has_veic) # define cpu_has_veic 0 #endif #ifndef cpu_has_inclusive_pcaches -#define cpu_has_inclusive_pcaches (cpu_data[0].options & MIPS_CPU_INCLUSIVE_CACHES) +#define cpu_has_inclusive_pcaches __opt(MIPS_CPU_INCLUSIVE_CACHES) #endif #ifndef cpu_dcache_line_size @@ -438,63 +476,63 @@ #endif #ifndef cpu_has_perf_cntr_intr_bit -#define cpu_has_perf_cntr_intr_bit (cpu_data[0].options & MIPS_CPU_PCI) +#define cpu_has_perf_cntr_intr_bit __opt(MIPS_CPU_PCI) #endif #ifndef cpu_has_vz -#define cpu_has_vz (cpu_data[0].ases & MIPS_ASE_VZ) +#define cpu_has_vz __ase(MIPS_ASE_VZ) #endif #if defined(CONFIG_CPU_HAS_MSA) && !defined(cpu_has_msa) -# define cpu_has_msa (cpu_data[0].ases & MIPS_ASE_MSA) +# define cpu_has_msa __ase(MIPS_ASE_MSA) #elif !defined(cpu_has_msa) # define cpu_has_msa 0 #endif #ifndef cpu_has_ufr -# define cpu_has_ufr (cpu_data[0].options & MIPS_CPU_UFR) +# define cpu_has_ufr __opt(MIPS_CPU_UFR) #endif #ifndef cpu_has_fre -# define cpu_has_fre (cpu_data[0].options & MIPS_CPU_FRE) +# define cpu_has_fre __opt(MIPS_CPU_FRE) #endif #ifndef cpu_has_cdmm -# define cpu_has_cdmm (cpu_data[0].options & MIPS_CPU_CDMM) +# define cpu_has_cdmm __opt(MIPS_CPU_CDMM) #endif #ifndef cpu_has_small_pages -# define cpu_has_small_pages (cpu_data[0].options & MIPS_CPU_SP) +# define cpu_has_small_pages __opt(MIPS_CPU_SP) #endif #ifndef cpu_has_nan_legacy -#define cpu_has_nan_legacy (cpu_data[0].options & MIPS_CPU_NAN_LEGACY) +#define cpu_has_nan_legacy __isa_lt_and_opt(6, MIPS_CPU_NAN_LEGACY) #endif #ifndef cpu_has_nan_2008 -#define cpu_has_nan_2008 (cpu_data[0].options & MIPS_CPU_NAN_2008) +#define cpu_has_nan_2008 __isa_ge_or_opt(6, MIPS_CPU_NAN_2008) #endif #ifndef cpu_has_ebase_wg -# define cpu_has_ebase_wg (cpu_data[0].options & MIPS_CPU_EBASE_WG) +# define cpu_has_ebase_wg __opt(MIPS_CPU_EBASE_WG) #endif #ifndef cpu_has_badinstr -# define cpu_has_badinstr (cpu_data[0].options & MIPS_CPU_BADINSTR) +# define cpu_has_badinstr __isa_ge_or_opt(6, MIPS_CPU_BADINSTR) #endif #ifndef cpu_has_badinstrp -# define cpu_has_badinstrp (cpu_data[0].options & MIPS_CPU_BADINSTRP) +# define cpu_has_badinstrp __isa_ge_or_opt(6, MIPS_CPU_BADINSTRP) #endif #ifndef cpu_has_contextconfig -# define cpu_has_contextconfig (cpu_data[0].options & MIPS_CPU_CTXTC) +# define cpu_has_contextconfig __opt(MIPS_CPU_CTXTC) #endif #ifndef cpu_has_perf -# define cpu_has_perf (cpu_data[0].options & MIPS_CPU_PERF) +# define cpu_has_perf __opt(MIPS_CPU_PERF) #endif -#if defined(CONFIG_SMP) && (MIPS_ISA_REV >= 6) +#ifdef CONFIG_SMP /* * Some systems share FTLB RAMs between threads within a core (siblings in * kernel parlance). This means that FTLB entries may become invalid at almost @@ -507,7 +545,7 @@ */ # ifndef cpu_has_shared_ftlb_ram # define cpu_has_shared_ftlb_ram \ - (current_cpu_data.options & MIPS_CPU_SHARED_FTLB_RAM) + __isa_ge_and_opt(6, MIPS_CPU_SHARED_FTLB_RAM) # endif /* @@ -524,9 +562,9 @@ */ # ifndef cpu_has_shared_ftlb_entries # define cpu_has_shared_ftlb_entries \ - (current_cpu_data.options & MIPS_CPU_SHARED_FTLB_ENTRIES) + __isa_ge_and_opt(6, MIPS_CPU_SHARED_FTLB_ENTRIES) # endif -#endif /* SMP && MIPS_ISA_REV >= 6 */ +#endif /* SMP */ #ifndef cpu_has_shared_ftlb_ram # define cpu_has_shared_ftlb_ram 0 @@ -537,7 +575,7 @@ #ifdef CONFIG_MIPS_MT_SMP # define cpu_has_mipsmt_pertccounters \ - (cpu_data[0].options & MIPS_CPU_MT_PER_TC_PERF_COUNTERS) + __isa_lt_and_opt(6, MIPS_CPU_MT_PER_TC_PERF_COUNTERS) #else # define cpu_has_mipsmt_pertccounters 0 #endif /* CONFIG_MIPS_MT_SMP */ diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index 5b9d02ef4f60..dacbdb84516a 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -225,31 +225,32 @@ * Definitions for 7:0 on legacy processors */ -#define PRID_REV_TX4927 0x0022 -#define PRID_REV_TX4937 0x0030 -#define PRID_REV_R4400 0x0040 -#define PRID_REV_R3000A 0x0030 -#define PRID_REV_R3000 0x0020 -#define PRID_REV_R2000A 0x0010 -#define PRID_REV_TX3912 0x0010 -#define PRID_REV_TX3922 0x0030 -#define PRID_REV_TX3927 0x0040 -#define PRID_REV_VR4111 0x0050 -#define PRID_REV_VR4181 0x0050 /* Same as VR4111 */ -#define PRID_REV_VR4121 0x0060 -#define PRID_REV_VR4122 0x0070 -#define PRID_REV_VR4181A 0x0070 /* Same as VR4122 */ -#define PRID_REV_VR4130 0x0080 -#define PRID_REV_34K_V1_0_2 0x0022 -#define PRID_REV_LOONGSON1B 0x0020 -#define PRID_REV_LOONGSON1C 0x0020 /* Same as Loongson-1B */ -#define PRID_REV_LOONGSON2E 0x0002 -#define PRID_REV_LOONGSON2F 0x0003 -#define PRID_REV_LOONGSON3A_R1 0x0005 -#define PRID_REV_LOONGSON3B_R1 0x0006 -#define PRID_REV_LOONGSON3B_R2 0x0007 -#define PRID_REV_LOONGSON3A_R2 0x0008 -#define PRID_REV_LOONGSON3A_R3 0x0009 +#define PRID_REV_TX4927 0x0022 +#define PRID_REV_TX4937 0x0030 +#define PRID_REV_R4400 0x0040 +#define PRID_REV_R3000A 0x0030 +#define PRID_REV_R3000 0x0020 +#define PRID_REV_R2000A 0x0010 +#define PRID_REV_TX3912 0x0010 +#define PRID_REV_TX3922 0x0030 +#define PRID_REV_TX3927 0x0040 +#define PRID_REV_VR4111 0x0050 +#define PRID_REV_VR4181 0x0050 /* Same as VR4111 */ +#define PRID_REV_VR4121 0x0060 +#define PRID_REV_VR4122 0x0070 +#define PRID_REV_VR4181A 0x0070 /* Same as VR4122 */ +#define PRID_REV_VR4130 0x0080 +#define PRID_REV_34K_V1_0_2 0x0022 +#define PRID_REV_LOONGSON1B 0x0020 +#define PRID_REV_LOONGSON1C 0x0020 /* Same as Loongson-1B */ +#define PRID_REV_LOONGSON2E 0x0002 +#define PRID_REV_LOONGSON2F 0x0003 +#define PRID_REV_LOONGSON3A_R1 0x0005 +#define PRID_REV_LOONGSON3B_R1 0x0006 +#define PRID_REV_LOONGSON3B_R2 0x0007 +#define PRID_REV_LOONGSON3A_R2 0x0008 +#define PRID_REV_LOONGSON3A_R3_0 0x0009 +#define PRID_REV_LOONGSON3A_R3_1 0x000d /* * Older processors used to encode processor version and revision in two diff --git a/arch/mips/include/asm/dma-coherence.h b/arch/mips/include/asm/dma-coherence.h index 72d0eab02afc..8eda48748ed5 100644 --- a/arch/mips/include/asm/dma-coherence.h +++ b/arch/mips/include/asm/dma-coherence.h @@ -21,10 +21,10 @@ enum coherent_io_user_state { extern enum coherent_io_user_state coherentio; extern int hw_coherentio; #else -#ifdef CONFIG_DMA_COHERENT -#define coherentio IO_COHERENCE_ENABLED -#else +#ifdef CONFIG_DMA_NONCOHERENT #define coherentio IO_COHERENCE_DISABLED +#else +#define coherentio IO_COHERENCE_ENABLED #endif #define hw_coherentio 0 #endif /* CONFIG_DMA_MAYBE_COHERENT */ diff --git a/arch/mips/include/asm/dma-direct.h b/arch/mips/include/asm/dma-direct.h index f32f15530aba..b5c240806e1b 100644 --- a/arch/mips/include/asm/dma-direct.h +++ b/arch/mips/include/asm/dma-direct.h @@ -1 +1,16 @@ -#include <asm/dma-coherence.h> +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MIPS_DMA_DIRECT_H +#define _MIPS_DMA_DIRECT_H 1 + +static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) +{ + if (!dev->dma_mask) + return false; + + return addr + size - 1 <= *dev->dma_mask; +} + +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr); +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr); + +#endif /* _MIPS_DMA_DIRECT_H */ diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h index 886e75a383f2..e81c4e97ff1a 100644 --- a/arch/mips/include/asm/dma-mapping.h +++ b/arch/mips/include/asm/dma-mapping.h @@ -2,19 +2,21 @@ #ifndef _ASM_DMA_MAPPING_H #define _ASM_DMA_MAPPING_H -#include <linux/scatterlist.h> -#include <asm/dma-coherence.h> -#include <asm/cache.h> +#include <linux/swiotlb.h> -#ifndef CONFIG_SGI_IP27 /* Kludge to fix 2.6.39 build for IP27 */ -#include <dma-coherence.h> -#endif - -extern const struct dma_map_ops *mips_dma_map_ops; +extern const struct dma_map_ops jazz_dma_ops; static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - return mips_dma_map_ops; +#if defined(CONFIG_MACH_JAZZ) + return &jazz_dma_ops; +#elif defined(CONFIG_SWIOTLB) + return &swiotlb_dma_ops; +#elif defined(CONFIG_DMA_NONCOHERENT_OPS) + return &dma_noncoherent_ops; +#else + return &dma_direct_ops; +#endif } #define arch_setup_dma_ops arch_setup_dma_ops diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index cea8ad864b3f..54c730aed327 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -12,6 +12,8 @@ #ifndef _ASM_IO_H #define _ASM_IO_H +#define ARCH_HAS_IOREMAP_WC + #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/types.h> @@ -141,14 +143,14 @@ static inline void * phys_to_virt(unsigned long address) /* * ISA I/O bus memory addresses are 1:1 with the physical address. */ -static inline unsigned long isa_virt_to_bus(volatile void * address) +static inline unsigned long isa_virt_to_bus(volatile void *address) { - return (unsigned long)address - PAGE_OFFSET; + return virt_to_phys(address); } -static inline void * isa_bus_to_virt(unsigned long address) +static inline void *isa_bus_to_virt(unsigned long address) { - return (void *)(address + PAGE_OFFSET); + return phys_to_virt(address); } #define isa_page_to_bus page_to_phys @@ -278,15 +280,25 @@ static inline void __iomem * __ioremap_mode(phys_addr_t offset, unsigned long si #define ioremap_cache ioremap_cachable /* - * These two are MIPS specific ioremap variant. ioremap_cacheable_cow - * requests a cachable mapping, ioremap_uncached_accelerated requests a - * mapping using the uncached accelerated mode which isn't supported on - * all processors. + * ioremap_wc - map bus memory into CPU space + * @offset: bus address of the memory + * @size: size of the resource to map + * + * ioremap_wc performs a platform specific sequence of operations to + * make bus memory CPU accessible via the readb/readw/readl/writeb/ + * writew/writel functions and the other mmio helpers. The returned + * address is not guaranteed to be usable directly as a virtual + * address. + * + * This version of ioremap ensures that the memory is marked uncachable + * but accelerated by means of write-combining feature. It is specifically + * useful for PCIe prefetchable windows, which may vastly improve a + * communications performance. If it was determined on boot stage, what + * CPU CCA doesn't support UCA, the method shall fall-back to the + * _CACHE_UNCACHED option (see cpu_probe() method). */ -#define ioremap_cacheable_cow(offset, size) \ - __ioremap_mode((offset), (size), _CACHE_CACHABLE_COW) -#define ioremap_uncached_accelerated(offset, size) \ - __ioremap_mode((offset), (size), _CACHE_UNCACHED_ACCELERATED) +#define ioremap_wc(offset, size) \ + __ioremap_mode((offset), (size), boot_cpu_data.writecombine) static inline void iounmap(const volatile void __iomem *addr) { @@ -590,7 +602,7 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int * * This API used to be exported; it now is for arch code internal use only. */ -#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) +#ifdef CONFIG_DMA_NONCOHERENT extern void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); extern void (*_dma_cache_wback)(unsigned long start, unsigned long size); @@ -609,7 +621,7 @@ extern void (*_dma_cache_inv)(unsigned long start, unsigned long size); #define dma_cache_inv(start,size) \ do { (void) (start); (void) (size); } while (0) -#endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */ +#endif /* CONFIG_DMA_NONCOHERENT */ /* * Read a 32-bit register that requires a 64-bit read cycle on the bus. diff --git a/arch/mips/include/asm/kprobes.h b/arch/mips/include/asm/kprobes.h index ad1a99948f27..a72dfbf1babb 100644 --- a/arch/mips/include/asm/kprobes.h +++ b/arch/mips/include/asm/kprobes.h @@ -68,16 +68,6 @@ struct prev_kprobe { unsigned long saved_epc; }; -#define MAX_JPROBES_STACK_SIZE 128 -#define MAX_JPROBES_STACK_ADDR \ - (((unsigned long)current_thread_info()) + THREAD_SIZE - 32 - sizeof(struct pt_regs)) - -#define MIN_JPROBES_STACK_SIZE(ADDR) \ - ((((ADDR) + MAX_JPROBES_STACK_SIZE) > MAX_JPROBES_STACK_ADDR) \ - ? MAX_JPROBES_STACK_ADDR - (ADDR) \ - : MAX_JPROBES_STACK_SIZE) - - #define SKIP_DELAYSLOT 0x0001 /* per-cpu kprobe control block */ @@ -86,12 +76,9 @@ struct kprobe_ctlblk { unsigned long kprobe_old_SR; unsigned long kprobe_saved_SR; unsigned long kprobe_saved_epc; - unsigned long jprobe_saved_sp; - struct pt_regs jprobe_saved_regs; /* Per-thread fields, used while emulating branches */ unsigned long flags; unsigned long target_epc; - u8 jprobes_stack[MAX_JPROBES_STACK_SIZE]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/mips/include/asm/mach-ar7/spaces.h b/arch/mips/include/asm/mach-ar7/spaces.h index 660ab64c0fc9..a004d94dfbdd 100644 --- a/arch/mips/include/asm/mach-ar7/spaces.h +++ b/arch/mips/include/asm/mach-ar7/spaces.h @@ -17,9 +17,6 @@ #define PAGE_OFFSET _AC(0x94000000, UL) #define PHYS_OFFSET _AC(0x14000000, UL) -#define UNCAC_BASE _AC(0xb4000000, UL) /* 0xa0000000 + PHYS_OFFSET */ -#define IO_BASE UNCAC_BASE - #include <asm/mach-generic/spaces.h> #endif /* __ASM_AR7_SPACES_H */ diff --git a/arch/mips/include/asm/mach-ath25/dma-coherence.h b/arch/mips/include/asm/mach-ath25/dma-coherence.h deleted file mode 100644 index d5defdde32db..000000000000 --- a/arch/mips/include/asm/mach-ath25/dma-coherence.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * Copyright (C) 2007 Felix Fietkau <nbd@openwrt.org> - * - */ -#ifndef __ASM_MACH_ATH25_DMA_COHERENCE_H -#define __ASM_MACH_ATH25_DMA_COHERENCE_H - -#include <linux/device.h> - -/* - * We need some arbitrary non-zero value to be programmed to the BAR1 register - * of PCI host controller to enable DMA. The same value should be used as the - * offset to calculate the physical address of DMA buffer for PCI devices. - */ -#define AR2315_PCI_HOST_SDRAM_BASEADDR 0x20000000 - -static inline dma_addr_t ath25_dev_offset(struct device *dev) -{ -#ifdef CONFIG_PCI - extern struct bus_type pci_bus_type; - - if (dev && dev->bus == &pci_bus_type) - return AR2315_PCI_HOST_SDRAM_BASEADDR; -#endif - return 0; -} - -static inline dma_addr_t -plat_map_dma_mem(struct device *dev, void *addr, size_t size) -{ - return virt_to_phys(addr) + ath25_dev_offset(dev); -} - -static inline dma_addr_t -plat_map_dma_mem_page(struct device *dev, struct page *page) -{ - return page_to_phys(page) + ath25_dev_offset(dev); -} - -static inline unsigned long -plat_dma_addr_to_phys(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr - ath25_dev_offset(dev); -} - -static inline void -plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - return 1; -} - -static inline int plat_device_is_coherent(struct device *dev) -{ -#ifdef CONFIG_DMA_COHERENT - return 1; -#endif -#ifdef CONFIG_DMA_NONCOHERENT - return 0; -#endif -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -#endif /* __ASM_MACH_ATH25_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-ath79/ar71xx_regs.h b/arch/mips/include/asm/mach-ath79/ar71xx_regs.h index d99ca862dae3..284b4fa23e03 100644 --- a/arch/mips/include/asm/mach-ath79/ar71xx_regs.h +++ b/arch/mips/include/asm/mach-ath79/ar71xx_regs.h @@ -20,6 +20,10 @@ #include <linux/bitops.h> #define AR71XX_APB_BASE 0x18000000 +#define AR71XX_GE0_BASE 0x19000000 +#define AR71XX_GE0_SIZE 0x10000 +#define AR71XX_GE1_BASE 0x1a000000 +#define AR71XX_GE1_SIZE 0x10000 #define AR71XX_EHCI_BASE 0x1b000000 #define AR71XX_EHCI_SIZE 0x1000 #define AR71XX_OHCI_BASE 0x1c000000 @@ -39,6 +43,8 @@ #define AR71XX_PLL_SIZE 0x100 #define AR71XX_RESET_BASE (AR71XX_APB_BASE + 0x00060000) #define AR71XX_RESET_SIZE 0x100 +#define AR71XX_MII_BASE (AR71XX_APB_BASE + 0x00070000) +#define AR71XX_MII_SIZE 0x100 #define AR71XX_PCI_MEM_BASE 0x10000000 #define AR71XX_PCI_MEM_SIZE 0x07000000 @@ -81,18 +87,39 @@ #define AR933X_UART_BASE (AR71XX_APB_BASE + 0x00020000) #define AR933X_UART_SIZE 0x14 +#define AR933X_GMAC_BASE (AR71XX_APB_BASE + 0x00070000) +#define AR933X_GMAC_SIZE 0x04 #define AR933X_WMAC_BASE (AR71XX_APB_BASE + 0x00100000) #define AR933X_WMAC_SIZE 0x20000 #define AR933X_EHCI_BASE 0x1b000000 #define AR933X_EHCI_SIZE 0x1000 +#define AR934X_GMAC_BASE (AR71XX_APB_BASE + 0x00070000) +#define AR934X_GMAC_SIZE 0x14 #define AR934X_WMAC_BASE (AR71XX_APB_BASE + 0x00100000) #define AR934X_WMAC_SIZE 0x20000 #define AR934X_EHCI_BASE 0x1b000000 #define AR934X_EHCI_SIZE 0x200 +#define AR934X_NFC_BASE 0x1b000200 +#define AR934X_NFC_SIZE 0xb8 #define AR934X_SRIF_BASE (AR71XX_APB_BASE + 0x00116000) #define AR934X_SRIF_SIZE 0x1000 +#define QCA953X_GMAC_BASE (AR71XX_APB_BASE + 0x00070000) +#define QCA953X_GMAC_SIZE 0x14 +#define QCA953X_WMAC_BASE (AR71XX_APB_BASE + 0x00100000) +#define QCA953X_WMAC_SIZE 0x20000 +#define QCA953X_EHCI_BASE 0x1b000000 +#define QCA953X_EHCI_SIZE 0x200 +#define QCA953X_SRIF_BASE (AR71XX_APB_BASE + 0x00116000) +#define QCA953X_SRIF_SIZE 0x1000 + +#define QCA953X_PCI_CFG_BASE0 0x14000000 +#define QCA953X_PCI_CTRL_BASE0 (AR71XX_APB_BASE + 0x000f0000) +#define QCA953X_PCI_CRP_BASE0 (AR71XX_APB_BASE + 0x000c0000) +#define QCA953X_PCI_MEM_BASE0 0x10000000 +#define QCA953X_PCI_MEM_SIZE 0x02000000 + #define QCA955X_PCI_MEM_BASE0 0x10000000 #define QCA955X_PCI_MEM_BASE1 0x12000000 #define QCA955X_PCI_MEM_SIZE 0x02000000 @@ -106,11 +133,72 @@ #define QCA955X_PCI_CTRL_BASE1 (AR71XX_APB_BASE + 0x00280000) #define QCA955X_PCI_CTRL_SIZE 0x100 +#define QCA955X_GMAC_BASE (AR71XX_APB_BASE + 0x00070000) +#define QCA955X_GMAC_SIZE 0x40 #define QCA955X_WMAC_BASE (AR71XX_APB_BASE + 0x00100000) #define QCA955X_WMAC_SIZE 0x20000 #define QCA955X_EHCI0_BASE 0x1b000000 #define QCA955X_EHCI1_BASE 0x1b400000 #define QCA955X_EHCI_SIZE 0x1000 +#define QCA955X_NFC_BASE 0x1b800200 +#define QCA955X_NFC_SIZE 0xb8 + +#define QCA956X_PCI_MEM_BASE1 0x12000000 +#define QCA956X_PCI_MEM_SIZE 0x02000000 +#define QCA956X_PCI_CFG_BASE1 0x16000000 +#define QCA956X_PCI_CFG_SIZE 0x1000 +#define QCA956X_PCI_CRP_BASE1 (AR71XX_APB_BASE + 0x00250000) +#define QCA956X_PCI_CRP_SIZE 0x1000 +#define QCA956X_PCI_CTRL_BASE1 (AR71XX_APB_BASE + 0x00280000) +#define QCA956X_PCI_CTRL_SIZE 0x100 + +#define QCA956X_WMAC_BASE (AR71XX_APB_BASE + 0x00100000) +#define QCA956X_WMAC_SIZE 0x20000 +#define QCA956X_EHCI0_BASE 0x1b000000 +#define QCA956X_EHCI1_BASE 0x1b400000 +#define QCA956X_EHCI_SIZE 0x200 +#define QCA956X_GMAC_SGMII_BASE (AR71XX_APB_BASE + 0x00070000) +#define QCA956X_GMAC_SGMII_SIZE 0x64 +#define QCA956X_PLL_BASE (AR71XX_APB_BASE + 0x00050000) +#define QCA956X_PLL_SIZE 0x50 +#define QCA956X_GMAC_BASE (AR71XX_APB_BASE + 0x00070000) +#define QCA956X_GMAC_SIZE 0x64 + +/* + * Hidden Registers + */ +#define QCA956X_MAC_CFG_BASE 0xb9000000 +#define QCA956X_MAC_CFG_SIZE 0x64 + +#define QCA956X_MAC_CFG1_REG 0x00 +#define QCA956X_MAC_CFG1_SOFT_RST BIT(31) +#define QCA956X_MAC_CFG1_RX_RST BIT(19) +#define QCA956X_MAC_CFG1_TX_RST BIT(18) +#define QCA956X_MAC_CFG1_LOOPBACK BIT(8) +#define QCA956X_MAC_CFG1_RX_EN BIT(2) +#define QCA956X_MAC_CFG1_TX_EN BIT(0) + +#define QCA956X_MAC_CFG2_REG 0x04 +#define QCA956X_MAC_CFG2_IF_1000 BIT(9) +#define QCA956X_MAC_CFG2_IF_10_100 BIT(8) +#define QCA956X_MAC_CFG2_HUGE_FRAME_EN BIT(5) +#define QCA956X_MAC_CFG2_LEN_CHECK BIT(4) +#define QCA956X_MAC_CFG2_PAD_CRC_EN BIT(2) +#define QCA956X_MAC_CFG2_FDX BIT(0) + +#define QCA956X_MAC_MII_MGMT_CFG_REG 0x20 +#define QCA956X_MGMT_CFG_CLK_DIV_20 0x07 + +#define QCA956X_MAC_FIFO_CFG0_REG 0x48 +#define QCA956X_MAC_FIFO_CFG1_REG 0x4c +#define QCA956X_MAC_FIFO_CFG2_REG 0x50 +#define QCA956X_MAC_FIFO_CFG3_REG 0x54 +#define QCA956X_MAC_FIFO_CFG4_REG 0x58 +#define QCA956X_MAC_FIFO_CFG5_REG 0x5c + +#define QCA956X_DAM_RESET_OFFSET 0xb90001bc +#define QCA956X_DAM_RESET_SIZE 0x4 +#define QCA956X_INLINE_CHKSUM_ENG BIT(27) /* * DDR_CTRL block @@ -149,6 +237,12 @@ #define AR934X_DDR_REG_FLUSH_PCIE 0xa8 #define AR934X_DDR_REG_FLUSH_WMAC 0xac +#define QCA953X_DDR_REG_FLUSH_GE0 0x9c +#define QCA953X_DDR_REG_FLUSH_GE1 0xa0 +#define QCA953X_DDR_REG_FLUSH_USB 0xa4 +#define QCA953X_DDR_REG_FLUSH_PCIE 0xa8 +#define QCA953X_DDR_REG_FLUSH_WMAC 0xac + /* * PLL block */ @@ -166,9 +260,15 @@ #define AR71XX_AHB_DIV_SHIFT 20 #define AR71XX_AHB_DIV_MASK 0x7 +#define AR71XX_ETH0_PLL_SHIFT 17 +#define AR71XX_ETH1_PLL_SHIFT 19 + #define AR724X_PLL_REG_CPU_CONFIG 0x00 #define AR724X_PLL_REG_PCIE_CONFIG 0x10 +#define AR724X_PLL_REG_PCIE_CONFIG_PPL_BYPASS BIT(16) +#define AR724X_PLL_REG_PCIE_CONFIG_PPL_RESET BIT(25) + #define AR724X_PLL_FB_SHIFT 0 #define AR724X_PLL_FB_MASK 0x3ff #define AR724X_PLL_REF_DIV_SHIFT 10 @@ -178,6 +278,8 @@ #define AR724X_DDR_DIV_SHIFT 22 #define AR724X_DDR_DIV_MASK 0x3 +#define AR7242_PLL_REG_ETH0_INT_CLOCK 0x2c + #define AR913X_PLL_REG_CPU_CONFIG 0x00 #define AR913X_PLL_REG_ETH_CONFIG 0x04 #define AR913X_PLL_REG_ETH0_INT_CLOCK 0x14 @@ -190,6 +292,9 @@ #define AR913X_AHB_DIV_SHIFT 19 #define AR913X_AHB_DIV_MASK 0x1 +#define AR913X_ETH0_PLL_SHIFT 20 +#define AR913X_ETH1_PLL_SHIFT 22 + #define AR933X_PLL_CPU_CONFIG_REG 0x00 #define AR933X_PLL_CLOCK_CTRL_REG 0x08 @@ -211,6 +316,8 @@ #define AR934X_PLL_CPU_CONFIG_REG 0x00 #define AR934X_PLL_DDR_CONFIG_REG 0x04 #define AR934X_PLL_CPU_DDR_CLK_CTRL_REG 0x08 +#define AR934X_PLL_SWITCH_CLOCK_CONTROL_REG 0x24 +#define AR934X_PLL_ETH_XMII_CONTROL_REG 0x2c #define AR934X_PLL_CPU_CONFIG_NFRAC_SHIFT 0 #define AR934X_PLL_CPU_CONFIG_NFRAC_MASK 0x3f @@ -243,9 +350,52 @@ #define AR934X_PLL_CPU_DDR_CLK_CTRL_DDRCLK_FROM_DDRPLL BIT(21) #define AR934X_PLL_CPU_DDR_CLK_CTRL_AHBCLK_FROM_DDRPLL BIT(24) +#define AR934X_PLL_SWITCH_CLOCK_CONTROL_MDIO_CLK_SEL BIT(6) + +#define QCA953X_PLL_CPU_CONFIG_REG 0x00 +#define QCA953X_PLL_DDR_CONFIG_REG 0x04 +#define QCA953X_PLL_CLK_CTRL_REG 0x08 +#define QCA953X_PLL_SWITCH_CLOCK_CONTROL_REG 0x24 +#define QCA953X_PLL_ETH_XMII_CONTROL_REG 0x2c +#define QCA953X_PLL_ETH_SGMII_CONTROL_REG 0x48 + +#define QCA953X_PLL_CPU_CONFIG_NFRAC_SHIFT 0 +#define QCA953X_PLL_CPU_CONFIG_NFRAC_MASK 0x3f +#define QCA953X_PLL_CPU_CONFIG_NINT_SHIFT 6 +#define QCA953X_PLL_CPU_CONFIG_NINT_MASK 0x3f +#define QCA953X_PLL_CPU_CONFIG_REFDIV_SHIFT 12 +#define QCA953X_PLL_CPU_CONFIG_REFDIV_MASK 0x1f +#define QCA953X_PLL_CPU_CONFIG_OUTDIV_SHIFT 19 +#define QCA953X_PLL_CPU_CONFIG_OUTDIV_MASK 0x7 + +#define QCA953X_PLL_DDR_CONFIG_NFRAC_SHIFT 0 +#define QCA953X_PLL_DDR_CONFIG_NFRAC_MASK 0x3ff +#define QCA953X_PLL_DDR_CONFIG_NINT_SHIFT 10 +#define QCA953X_PLL_DDR_CONFIG_NINT_MASK 0x3f +#define QCA953X_PLL_DDR_CONFIG_REFDIV_SHIFT 16 +#define QCA953X_PLL_DDR_CONFIG_REFDIV_MASK 0x1f +#define QCA953X_PLL_DDR_CONFIG_OUTDIV_SHIFT 23 +#define QCA953X_PLL_DDR_CONFIG_OUTDIV_MASK 0x7 + +#define QCA953X_PLL_CLK_CTRL_CPU_PLL_BYPASS BIT(2) +#define QCA953X_PLL_CLK_CTRL_DDR_PLL_BYPASS BIT(3) +#define QCA953X_PLL_CLK_CTRL_AHB_PLL_BYPASS BIT(4) +#define QCA953X_PLL_CLK_CTRL_CPU_POST_DIV_SHIFT 5 +#define QCA953X_PLL_CLK_CTRL_CPU_POST_DIV_MASK 0x1f +#define QCA953X_PLL_CLK_CTRL_DDR_POST_DIV_SHIFT 10 +#define QCA953X_PLL_CLK_CTRL_DDR_POST_DIV_MASK 0x1f +#define QCA953X_PLL_CLK_CTRL_AHB_POST_DIV_SHIFT 15 +#define QCA953X_PLL_CLK_CTRL_AHB_POST_DIV_MASK 0x1f +#define QCA953X_PLL_CLK_CTRL_CPUCLK_FROM_CPUPLL BIT(20) +#define QCA953X_PLL_CLK_CTRL_DDRCLK_FROM_DDRPLL BIT(21) +#define QCA953X_PLL_CLK_CTRL_AHBCLK_FROM_DDRPLL BIT(24) + #define QCA955X_PLL_CPU_CONFIG_REG 0x00 #define QCA955X_PLL_DDR_CONFIG_REG 0x04 #define QCA955X_PLL_CLK_CTRL_REG 0x08 +#define QCA955X_PLL_ETH_XMII_CONTROL_REG 0x28 +#define QCA955X_PLL_ETH_SGMII_CONTROL_REG 0x48 +#define QCA955X_PLL_ETH_SGMII_SERDES_REG 0x4c #define QCA955X_PLL_CPU_CONFIG_NFRAC_SHIFT 0 #define QCA955X_PLL_CPU_CONFIG_NFRAC_MASK 0x3f @@ -278,6 +428,81 @@ #define QCA955X_PLL_CLK_CTRL_DDRCLK_FROM_DDRPLL BIT(21) #define QCA955X_PLL_CLK_CTRL_AHBCLK_FROM_DDRPLL BIT(24) +#define QCA955X_PLL_ETH_SGMII_SERDES_LOCK_DETECT BIT(2) +#define QCA955X_PLL_ETH_SGMII_SERDES_PLL_REFCLK BIT(1) +#define QCA955X_PLL_ETH_SGMII_SERDES_EN_PLL BIT(0) + +#define QCA956X_PLL_CPU_CONFIG_REG 0x00 +#define QCA956X_PLL_CPU_CONFIG1_REG 0x04 +#define QCA956X_PLL_DDR_CONFIG_REG 0x08 +#define QCA956X_PLL_DDR_CONFIG1_REG 0x0c +#define QCA956X_PLL_CLK_CTRL_REG 0x10 +#define QCA956X_PLL_SWITCH_CLOCK_CONTROL_REG 0x28 +#define QCA956X_PLL_ETH_XMII_CONTROL_REG 0x30 +#define QCA956X_PLL_ETH_SGMII_SERDES_REG 0x4c + +#define QCA956X_PLL_CPU_CONFIG_REFDIV_SHIFT 12 +#define QCA956X_PLL_CPU_CONFIG_REFDIV_MASK 0x1f +#define QCA956X_PLL_CPU_CONFIG_OUTDIV_SHIFT 19 +#define QCA956X_PLL_CPU_CONFIG_OUTDIV_MASK 0x7 + +#define QCA956X_PLL_CPU_CONFIG1_NFRAC_L_SHIFT 0 +#define QCA956X_PLL_CPU_CONFIG1_NFRAC_L_MASK 0x1f +#define QCA956X_PLL_CPU_CONFIG1_NFRAC_H_SHIFT 5 +#define QCA956X_PLL_CPU_CONFIG1_NFRAC_H_MASK 0x1fff +#define QCA956X_PLL_CPU_CONFIG1_NINT_SHIFT 18 +#define QCA956X_PLL_CPU_CONFIG1_NINT_MASK 0x1ff + +#define QCA956X_PLL_DDR_CONFIG_REFDIV_SHIFT 16 +#define QCA956X_PLL_DDR_CONFIG_REFDIV_MASK 0x1f +#define QCA956X_PLL_DDR_CONFIG_OUTDIV_SHIFT 23 +#define QCA956X_PLL_DDR_CONFIG_OUTDIV_MASK 0x7 + +#define QCA956X_PLL_DDR_CONFIG1_NFRAC_L_SHIFT 0 +#define QCA956X_PLL_DDR_CONFIG1_NFRAC_L_MASK 0x1f +#define QCA956X_PLL_DDR_CONFIG1_NFRAC_H_SHIFT 5 +#define QCA956X_PLL_DDR_CONFIG1_NFRAC_H_MASK 0x1fff +#define QCA956X_PLL_DDR_CONFIG1_NINT_SHIFT 18 +#define QCA956X_PLL_DDR_CONFIG1_NINT_MASK 0x1ff + +#define QCA956X_PLL_CLK_CTRL_CPU_PLL_BYPASS BIT(2) +#define QCA956X_PLL_CLK_CTRL_DDR_PLL_BYPASS BIT(3) +#define QCA956X_PLL_CLK_CTRL_AHB_PLL_BYPASS BIT(4) +#define QCA956X_PLL_CLK_CTRL_CPU_POST_DIV_SHIFT 5 +#define QCA956X_PLL_CLK_CTRL_CPU_POST_DIV_MASK 0x1f +#define QCA956X_PLL_CLK_CTRL_DDR_POST_DIV_SHIFT 10 +#define QCA956X_PLL_CLK_CTRL_DDR_POST_DIV_MASK 0x1f +#define QCA956X_PLL_CLK_CTRL_AHB_POST_DIV_SHIFT 15 +#define QCA956X_PLL_CLK_CTRL_AHB_POST_DIV_MASK 0x1f +#define QCA956X_PLL_CLK_CTRL_CPU_DDRCLK_FROM_DDRPLL BIT(20) +#define QCA956X_PLL_CLK_CTRL_CPU_DDRCLK_FROM_CPUPLL BIT(21) +#define QCA956X_PLL_CLK_CTRL_AHBCLK_FROM_DDRPLL BIT(24) + +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_I2C_CLK_SELB BIT(5) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_MDIO_CLK_SEL0_1 BIT(6) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_UART1_CLK_SEL BIT(7) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_USB_REFCLK_FREQ_SEL_SHIFT 8 +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_USB_REFCLK_FREQ_SEL_MASK 0xf +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_EN_PLL_TOP BIT(12) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_MDIO_CLK_SEL0_2 BIT(13) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_MDIO_CLK_SEL1_1 BIT(14) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_MDIO_CLK_SEL1_2 BIT(15) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_SWITCH_FUNC_TST_MODE BIT(16) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_EEE_ENABLE BIT(17) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_OEN_CLK125M_PLL BIT(18) +#define QCA956X_PLL_SWITCH_CLOCK_SPARE_SWITCHCLK_SEL BIT(19) + +#define QCA956X_PLL_ETH_XMII_TX_INVERT BIT(1) +#define QCA956X_PLL_ETH_XMII_GIGE BIT(25) +#define QCA956X_PLL_ETH_XMII_RX_DELAY_SHIFT 28 +#define QCA956X_PLL_ETH_XMII_RX_DELAY_MASK 0x3 +#define QCA956X_PLL_ETH_XMII_TX_DELAY_SHIFT 26 +#define QCA956X_PLL_ETH_XMII_TX_DELAY_MASK 3 + +#define QCA956X_PLL_ETH_SGMII_SERDES_LOCK_DETECT BIT(2) +#define QCA956X_PLL_ETH_SGMII_SERDES_PLL_REFCLK BIT(1) +#define QCA956X_PLL_ETH_SGMII_SERDES_EN_PLL BIT(0) + /* * USB_CONFIG block */ @@ -317,10 +542,19 @@ #define AR934X_RESET_REG_BOOTSTRAP 0xb0 #define AR934X_RESET_REG_PCIE_WMAC_INT_STATUS 0xac +#define QCA953X_RESET_REG_RESET_MODULE 0x1c +#define QCA953X_RESET_REG_BOOTSTRAP 0xb0 +#define QCA953X_RESET_REG_PCIE_WMAC_INT_STATUS 0xac + #define QCA955X_RESET_REG_RESET_MODULE 0x1c #define QCA955X_RESET_REG_BOOTSTRAP 0xb0 #define QCA955X_RESET_REG_EXT_INT_STATUS 0xac +#define QCA956X_RESET_REG_RESET_MODULE 0x1c +#define QCA956X_RESET_REG_BOOTSTRAP 0xb0 +#define QCA956X_RESET_REG_EXT_INT_STATUS 0xac + +#define MISC_INT_MIPS_SI_TIMERINT_MASK BIT(28) #define MISC_INT_ETHSW BIT(12) #define MISC_INT_TIMER4 BIT(10) #define MISC_INT_TIMER3 BIT(9) @@ -370,16 +604,123 @@ #define AR913X_RESET_USB_HOST BIT(5) #define AR913X_RESET_USB_PHY BIT(4) +#define AR933X_RESET_GE1_MDIO BIT(23) +#define AR933X_RESET_GE0_MDIO BIT(22) +#define AR933X_RESET_GE1_MAC BIT(13) #define AR933X_RESET_WMAC BIT(11) +#define AR933X_RESET_GE0_MAC BIT(9) #define AR933X_RESET_USB_HOST BIT(5) #define AR933X_RESET_USB_PHY BIT(4) #define AR933X_RESET_USBSUS_OVERRIDE BIT(3) +#define AR934X_RESET_HOST BIT(31) +#define AR934X_RESET_SLIC BIT(30) +#define AR934X_RESET_HDMA BIT(29) +#define AR934X_RESET_EXTERNAL BIT(28) +#define AR934X_RESET_RTC BIT(27) +#define AR934X_RESET_PCIE_EP_INT BIT(26) +#define AR934X_RESET_CHKSUM_ACC BIT(25) +#define AR934X_RESET_FULL_CHIP BIT(24) +#define AR934X_RESET_GE1_MDIO BIT(23) +#define AR934X_RESET_GE0_MDIO BIT(22) +#define AR934X_RESET_CPU_NMI BIT(21) +#define AR934X_RESET_CPU_COLD BIT(20) +#define AR934X_RESET_HOST_RESET_INT BIT(19) +#define AR934X_RESET_PCIE_EP BIT(18) +#define AR934X_RESET_UART1 BIT(17) +#define AR934X_RESET_DDR BIT(16) +#define AR934X_RESET_USB_PHY_PLL_PWD_EXT BIT(15) +#define AR934X_RESET_NANDF BIT(14) +#define AR934X_RESET_GE1_MAC BIT(13) +#define AR934X_RESET_ETH_SWITCH_ANALOG BIT(12) #define AR934X_RESET_USB_PHY_ANALOG BIT(11) +#define AR934X_RESET_HOST_DMA_INT BIT(10) +#define AR934X_RESET_GE0_MAC BIT(9) +#define AR934X_RESET_ETH_SWITCH BIT(8) +#define AR934X_RESET_PCIE_PHY BIT(7) +#define AR934X_RESET_PCIE BIT(6) #define AR934X_RESET_USB_HOST BIT(5) #define AR934X_RESET_USB_PHY BIT(4) #define AR934X_RESET_USBSUS_OVERRIDE BIT(3) - +#define AR934X_RESET_LUT BIT(2) +#define AR934X_RESET_MBOX BIT(1) +#define AR934X_RESET_I2S BIT(0) + +#define QCA953X_RESET_USB_EXT_PWR BIT(29) +#define QCA953X_RESET_EXTERNAL BIT(28) +#define QCA953X_RESET_RTC BIT(27) +#define QCA953X_RESET_FULL_CHIP BIT(24) +#define QCA953X_RESET_GE1_MDIO BIT(23) +#define QCA953X_RESET_GE0_MDIO BIT(22) +#define QCA953X_RESET_CPU_NMI BIT(21) +#define QCA953X_RESET_CPU_COLD BIT(20) +#define QCA953X_RESET_DDR BIT(16) +#define QCA953X_RESET_USB_PHY_PLL_PWD_EXT BIT(15) +#define QCA953X_RESET_GE1_MAC BIT(13) +#define QCA953X_RESET_ETH_SWITCH_ANALOG BIT(12) +#define QCA953X_RESET_USB_PHY_ANALOG BIT(11) +#define QCA953X_RESET_GE0_MAC BIT(9) +#define QCA953X_RESET_ETH_SWITCH BIT(8) +#define QCA953X_RESET_PCIE_PHY BIT(7) +#define QCA953X_RESET_PCIE BIT(6) +#define QCA953X_RESET_USB_HOST BIT(5) +#define QCA953X_RESET_USB_PHY BIT(4) +#define QCA953X_RESET_USBSUS_OVERRIDE BIT(3) + +#define QCA955X_RESET_HOST BIT(31) +#define QCA955X_RESET_SLIC BIT(30) +#define QCA955X_RESET_HDMA BIT(29) +#define QCA955X_RESET_EXTERNAL BIT(28) +#define QCA955X_RESET_RTC BIT(27) +#define QCA955X_RESET_PCIE_EP_INT BIT(26) +#define QCA955X_RESET_CHKSUM_ACC BIT(25) +#define QCA955X_RESET_FULL_CHIP BIT(24) +#define QCA955X_RESET_GE1_MDIO BIT(23) +#define QCA955X_RESET_GE0_MDIO BIT(22) +#define QCA955X_RESET_CPU_NMI BIT(21) +#define QCA955X_RESET_CPU_COLD BIT(20) +#define QCA955X_RESET_HOST_RESET_INT BIT(19) +#define QCA955X_RESET_PCIE_EP BIT(18) +#define QCA955X_RESET_UART1 BIT(17) +#define QCA955X_RESET_DDR BIT(16) +#define QCA955X_RESET_USB_PHY_PLL_PWD_EXT BIT(15) +#define QCA955X_RESET_NANDF BIT(14) +#define QCA955X_RESET_GE1_MAC BIT(13) +#define QCA955X_RESET_SGMII_ANALOG BIT(12) +#define QCA955X_RESET_USB_PHY_ANALOG BIT(11) +#define QCA955X_RESET_HOST_DMA_INT BIT(10) +#define QCA955X_RESET_GE0_MAC BIT(9) +#define QCA955X_RESET_SGMII BIT(8) +#define QCA955X_RESET_PCIE_PHY BIT(7) +#define QCA955X_RESET_PCIE BIT(6) +#define QCA955X_RESET_USB_HOST BIT(5) +#define QCA955X_RESET_USB_PHY BIT(4) +#define QCA955X_RESET_USBSUS_OVERRIDE BIT(3) +#define QCA955X_RESET_LUT BIT(2) +#define QCA955X_RESET_MBOX BIT(1) +#define QCA955X_RESET_I2S BIT(0) + +#define QCA956X_RESET_EXTERNAL BIT(28) +#define QCA956X_RESET_FULL_CHIP BIT(24) +#define QCA956X_RESET_GE1_MDIO BIT(23) +#define QCA956X_RESET_GE0_MDIO BIT(22) +#define QCA956X_RESET_CPU_NMI BIT(21) +#define QCA956X_RESET_CPU_COLD BIT(20) +#define QCA956X_RESET_DMA BIT(19) +#define QCA956X_RESET_DDR BIT(16) +#define QCA956X_RESET_GE1_MAC BIT(13) +#define QCA956X_RESET_SGMII_ANALOG BIT(12) +#define QCA956X_RESET_USB_PHY_ANALOG BIT(11) +#define QCA956X_RESET_GE0_MAC BIT(9) +#define QCA956X_RESET_SGMII BIT(8) +#define QCA956X_RESET_USB_HOST BIT(5) +#define QCA956X_RESET_USB_PHY BIT(4) +#define QCA956X_RESET_USBSUS_OVERRIDE BIT(3) +#define QCA956X_RESET_SWITCH_ANALOG BIT(2) +#define QCA956X_RESET_SWITCH BIT(0) + +#define AR933X_BOOTSTRAP_MDIO_GPIO_EN BIT(18) +#define AR933X_BOOTSTRAP_EEPBUSY BIT(4) #define AR933X_BOOTSTRAP_REF_CLK_40 BIT(0) #define AR934X_BOOTSTRAP_SW_OPTION8 BIT(23) @@ -398,8 +739,17 @@ #define AR934X_BOOTSTRAP_SDRAM_DISABLED BIT(1) #define AR934X_BOOTSTRAP_DDR1 BIT(0) +#define QCA953X_BOOTSTRAP_SW_OPTION2 BIT(12) +#define QCA953X_BOOTSTRAP_SW_OPTION1 BIT(11) +#define QCA953X_BOOTSTRAP_EJTAG_MODE BIT(5) +#define QCA953X_BOOTSTRAP_REF_CLK_40 BIT(4) +#define QCA953X_BOOTSTRAP_SDRAM_DISABLED BIT(1) +#define QCA953X_BOOTSTRAP_DDR1 BIT(0) + #define QCA955X_BOOTSTRAP_REF_CLK_40 BIT(4) +#define QCA956X_BOOTSTRAP_REF_CLK_40 BIT(2) + #define AR934X_PCIE_WMAC_INT_WMAC_MISC BIT(0) #define AR934X_PCIE_WMAC_INT_WMAC_TX BIT(1) #define AR934X_PCIE_WMAC_INT_WMAC_RXLP BIT(2) @@ -418,6 +768,24 @@ AR934X_PCIE_WMAC_INT_PCIE_RC1 | AR934X_PCIE_WMAC_INT_PCIE_RC2 | \ AR934X_PCIE_WMAC_INT_PCIE_RC3) +#define QCA953X_PCIE_WMAC_INT_WMAC_MISC BIT(0) +#define QCA953X_PCIE_WMAC_INT_WMAC_TX BIT(1) +#define QCA953X_PCIE_WMAC_INT_WMAC_RXLP BIT(2) +#define QCA953X_PCIE_WMAC_INT_WMAC_RXHP BIT(3) +#define QCA953X_PCIE_WMAC_INT_PCIE_RC BIT(4) +#define QCA953X_PCIE_WMAC_INT_PCIE_RC0 BIT(5) +#define QCA953X_PCIE_WMAC_INT_PCIE_RC1 BIT(6) +#define QCA953X_PCIE_WMAC_INT_PCIE_RC2 BIT(7) +#define QCA953X_PCIE_WMAC_INT_PCIE_RC3 BIT(8) +#define QCA953X_PCIE_WMAC_INT_WMAC_ALL \ + (QCA953X_PCIE_WMAC_INT_WMAC_MISC | QCA953X_PCIE_WMAC_INT_WMAC_TX | \ + QCA953X_PCIE_WMAC_INT_WMAC_RXLP | QCA953X_PCIE_WMAC_INT_WMAC_RXHP) + +#define QCA953X_PCIE_WMAC_INT_PCIE_ALL \ + (QCA953X_PCIE_WMAC_INT_PCIE_RC | QCA953X_PCIE_WMAC_INT_PCIE_RC0 | \ + QCA953X_PCIE_WMAC_INT_PCIE_RC1 | QCA953X_PCIE_WMAC_INT_PCIE_RC2 | \ + QCA953X_PCIE_WMAC_INT_PCIE_RC3) + #define QCA955X_EXT_INT_WMAC_MISC BIT(0) #define QCA955X_EXT_INT_WMAC_TX BIT(1) #define QCA955X_EXT_INT_WMAC_RXLP BIT(2) @@ -449,6 +817,37 @@ QCA955X_EXT_INT_PCIE_RC2_INT1 | QCA955X_EXT_INT_PCIE_RC2_INT2 | \ QCA955X_EXT_INT_PCIE_RC2_INT3) +#define QCA956X_EXT_INT_WMAC_MISC BIT(0) +#define QCA956X_EXT_INT_WMAC_TX BIT(1) +#define QCA956X_EXT_INT_WMAC_RXLP BIT(2) +#define QCA956X_EXT_INT_WMAC_RXHP BIT(3) +#define QCA956X_EXT_INT_PCIE_RC1 BIT(4) +#define QCA956X_EXT_INT_PCIE_RC1_INT0 BIT(5) +#define QCA956X_EXT_INT_PCIE_RC1_INT1 BIT(6) +#define QCA956X_EXT_INT_PCIE_RC1_INT2 BIT(7) +#define QCA956X_EXT_INT_PCIE_RC1_INT3 BIT(8) +#define QCA956X_EXT_INT_PCIE_RC2 BIT(12) +#define QCA956X_EXT_INT_PCIE_RC2_INT0 BIT(13) +#define QCA956X_EXT_INT_PCIE_RC2_INT1 BIT(14) +#define QCA956X_EXT_INT_PCIE_RC2_INT2 BIT(15) +#define QCA956X_EXT_INT_PCIE_RC2_INT3 BIT(16) +#define QCA956X_EXT_INT_USB1 BIT(24) +#define QCA956X_EXT_INT_USB2 BIT(28) + +#define QCA956X_EXT_INT_WMAC_ALL \ + (QCA956X_EXT_INT_WMAC_MISC | QCA956X_EXT_INT_WMAC_TX | \ + QCA956X_EXT_INT_WMAC_RXLP | QCA956X_EXT_INT_WMAC_RXHP) + +#define QCA956X_EXT_INT_PCIE_RC1_ALL \ + (QCA956X_EXT_INT_PCIE_RC1 | QCA956X_EXT_INT_PCIE_RC1_INT0 | \ + QCA956X_EXT_INT_PCIE_RC1_INT1 | QCA956X_EXT_INT_PCIE_RC1_INT2 | \ + QCA956X_EXT_INT_PCIE_RC1_INT3) + +#define QCA956X_EXT_INT_PCIE_RC2_ALL \ + (QCA956X_EXT_INT_PCIE_RC2 | QCA956X_EXT_INT_PCIE_RC2_INT0 | \ + QCA956X_EXT_INT_PCIE_RC2_INT1 | QCA956X_EXT_INT_PCIE_RC2_INT2 | \ + QCA956X_EXT_INT_PCIE_RC2_INT3) + #define REV_ID_MAJOR_MASK 0xfff0 #define REV_ID_MAJOR_AR71XX 0x00a0 #define REV_ID_MAJOR_AR913X 0x00b0 @@ -460,8 +859,12 @@ #define REV_ID_MAJOR_AR9341 0x0120 #define REV_ID_MAJOR_AR9342 0x1120 #define REV_ID_MAJOR_AR9344 0x2120 +#define REV_ID_MAJOR_QCA9533 0x0140 +#define REV_ID_MAJOR_QCA9533_V2 0x0160 #define REV_ID_MAJOR_QCA9556 0x0130 #define REV_ID_MAJOR_QCA9558 0x1130 +#define REV_ID_MAJOR_TP9343 0x0150 +#define REV_ID_MAJOR_QCA956X 0x1150 #define AR71XX_REV_ID_MINOR_MASK 0x3 #define AR71XX_REV_ID_MINOR_AR7130 0x0 @@ -482,8 +885,12 @@ #define AR934X_REV_ID_REVISION_MASK 0xf +#define QCA953X_REV_ID_REVISION_MASK 0xf + #define QCA955X_REV_ID_REVISION_MASK 0xf +#define QCA956X_REV_ID_REVISION_MASK 0xf + /* * SPI block */ @@ -521,15 +928,63 @@ #define AR71XX_GPIO_REG_INT_ENABLE 0x24 #define AR71XX_GPIO_REG_FUNC 0x28 +#define AR934X_GPIO_REG_OUT_FUNC0 0x2c +#define AR934X_GPIO_REG_OUT_FUNC1 0x30 +#define AR934X_GPIO_REG_OUT_FUNC2 0x34 +#define AR934X_GPIO_REG_OUT_FUNC3 0x38 +#define AR934X_GPIO_REG_OUT_FUNC4 0x3c +#define AR934X_GPIO_REG_OUT_FUNC5 0x40 #define AR934X_GPIO_REG_FUNC 0x6c +#define QCA953X_GPIO_REG_OUT_FUNC0 0x2c +#define QCA953X_GPIO_REG_OUT_FUNC1 0x30 +#define QCA953X_GPIO_REG_OUT_FUNC2 0x34 +#define QCA953X_GPIO_REG_OUT_FUNC3 0x38 +#define QCA953X_GPIO_REG_OUT_FUNC4 0x3c +#define QCA953X_GPIO_REG_IN_ENABLE0 0x44 +#define QCA953X_GPIO_REG_FUNC 0x6c + +#define QCA953X_GPIO_OUT_MUX_SPI_CS1 10 +#define QCA953X_GPIO_OUT_MUX_SPI_CS2 11 +#define QCA953X_GPIO_OUT_MUX_SPI_CS0 9 +#define QCA953X_GPIO_OUT_MUX_SPI_CLK 8 +#define QCA953X_GPIO_OUT_MUX_SPI_MOSI 12 +#define QCA953X_GPIO_OUT_MUX_LED_LINK1 41 +#define QCA953X_GPIO_OUT_MUX_LED_LINK2 42 +#define QCA953X_GPIO_OUT_MUX_LED_LINK3 43 +#define QCA953X_GPIO_OUT_MUX_LED_LINK4 44 +#define QCA953X_GPIO_OUT_MUX_LED_LINK5 45 + +#define QCA955X_GPIO_REG_OUT_FUNC0 0x2c +#define QCA955X_GPIO_REG_OUT_FUNC1 0x30 +#define QCA955X_GPIO_REG_OUT_FUNC2 0x34 +#define QCA955X_GPIO_REG_OUT_FUNC3 0x38 +#define QCA955X_GPIO_REG_OUT_FUNC4 0x3c +#define QCA955X_GPIO_REG_OUT_FUNC5 0x40 +#define QCA955X_GPIO_REG_FUNC 0x6c + +#define QCA956X_GPIO_REG_OUT_FUNC0 0x2c +#define QCA956X_GPIO_REG_OUT_FUNC1 0x30 +#define QCA956X_GPIO_REG_OUT_FUNC2 0x34 +#define QCA956X_GPIO_REG_OUT_FUNC3 0x38 +#define QCA956X_GPIO_REG_OUT_FUNC4 0x3c +#define QCA956X_GPIO_REG_OUT_FUNC5 0x40 +#define QCA956X_GPIO_REG_IN_ENABLE0 0x44 +#define QCA956X_GPIO_REG_IN_ENABLE3 0x50 +#define QCA956X_GPIO_REG_FUNC 0x6c + +#define QCA956X_GPIO_OUT_MUX_GE0_MDO 32 +#define QCA956X_GPIO_OUT_MUX_GE0_MDC 33 + #define AR71XX_GPIO_COUNT 16 #define AR7240_GPIO_COUNT 18 #define AR7241_GPIO_COUNT 20 #define AR913X_GPIO_COUNT 22 #define AR933X_GPIO_COUNT 30 #define AR934X_GPIO_COUNT 23 +#define QCA953X_GPIO_COUNT 18 #define QCA955X_GPIO_COUNT 24 +#define QCA956X_GPIO_COUNT 23 /* * SRIF block @@ -552,4 +1007,318 @@ #define AR934X_SRIF_DPLL2_OUTDIV_SHIFT 13 #define AR934X_SRIF_DPLL2_OUTDIV_MASK 0x7 +#define QCA953X_SRIF_CPU_DPLL1_REG 0x1c0 +#define QCA953X_SRIF_CPU_DPLL2_REG 0x1c4 +#define QCA953X_SRIF_CPU_DPLL3_REG 0x1c8 + +#define QCA953X_SRIF_DDR_DPLL1_REG 0x240 +#define QCA953X_SRIF_DDR_DPLL2_REG 0x244 +#define QCA953X_SRIF_DDR_DPLL3_REG 0x248 + +#define QCA953X_SRIF_DPLL1_REFDIV_SHIFT 27 +#define QCA953X_SRIF_DPLL1_REFDIV_MASK 0x1f +#define QCA953X_SRIF_DPLL1_NINT_SHIFT 18 +#define QCA953X_SRIF_DPLL1_NINT_MASK 0x1ff +#define QCA953X_SRIF_DPLL1_NFRAC_MASK 0x0003ffff + +#define QCA953X_SRIF_DPLL2_LOCAL_PLL BIT(30) +#define QCA953X_SRIF_DPLL2_OUTDIV_SHIFT 13 +#define QCA953X_SRIF_DPLL2_OUTDIV_MASK 0x7 + +#define AR71XX_GPIO_FUNC_STEREO_EN BIT(17) +#define AR71XX_GPIO_FUNC_SLIC_EN BIT(16) +#define AR71XX_GPIO_FUNC_SPI_CS2_EN BIT(13) +#define AR71XX_GPIO_FUNC_SPI_CS1_EN BIT(12) +#define AR71XX_GPIO_FUNC_UART_EN BIT(8) +#define AR71XX_GPIO_FUNC_USB_OC_EN BIT(4) +#define AR71XX_GPIO_FUNC_USB_CLK_EN BIT(0) + +#define AR724X_GPIO_FUNC_GE0_MII_CLK_EN BIT(19) +#define AR724X_GPIO_FUNC_SPI_EN BIT(18) +#define AR724X_GPIO_FUNC_SPI_CS_EN2 BIT(14) +#define AR724X_GPIO_FUNC_SPI_CS_EN1 BIT(13) +#define AR724X_GPIO_FUNC_CLK_OBS5_EN BIT(12) +#define AR724X_GPIO_FUNC_CLK_OBS4_EN BIT(11) +#define AR724X_GPIO_FUNC_CLK_OBS3_EN BIT(10) +#define AR724X_GPIO_FUNC_CLK_OBS2_EN BIT(9) +#define AR724X_GPIO_FUNC_CLK_OBS1_EN BIT(8) +#define AR724X_GPIO_FUNC_ETH_SWITCH_LED4_EN BIT(7) +#define AR724X_GPIO_FUNC_ETH_SWITCH_LED3_EN BIT(6) +#define AR724X_GPIO_FUNC_ETH_SWITCH_LED2_EN BIT(5) +#define AR724X_GPIO_FUNC_ETH_SWITCH_LED1_EN BIT(4) +#define AR724X_GPIO_FUNC_ETH_SWITCH_LED0_EN BIT(3) +#define AR724X_GPIO_FUNC_UART_RTS_CTS_EN BIT(2) +#define AR724X_GPIO_FUNC_UART_EN BIT(1) +#define AR724X_GPIO_FUNC_JTAG_DISABLE BIT(0) + +#define AR913X_GPIO_FUNC_WMAC_LED_EN BIT(22) +#define AR913X_GPIO_FUNC_EXP_PORT_CS_EN BIT(21) +#define AR913X_GPIO_FUNC_I2S_REFCLKEN BIT(20) +#define AR913X_GPIO_FUNC_I2S_MCKEN BIT(19) +#define AR913X_GPIO_FUNC_I2S1_EN BIT(18) +#define AR913X_GPIO_FUNC_I2S0_EN BIT(17) +#define AR913X_GPIO_FUNC_SLIC_EN BIT(16) +#define AR913X_GPIO_FUNC_UART_RTSCTS_EN BIT(9) +#define AR913X_GPIO_FUNC_UART_EN BIT(8) +#define AR913X_GPIO_FUNC_USB_CLK_EN BIT(4) + +#define AR933X_GPIO_FUNC_SPDIF2TCK BIT(31) +#define AR933X_GPIO_FUNC_SPDIF_EN BIT(30) +#define AR933X_GPIO_FUNC_I2SO_22_18_EN BIT(29) +#define AR933X_GPIO_FUNC_I2S_MCK_EN BIT(27) +#define AR933X_GPIO_FUNC_I2SO_EN BIT(26) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED_DUPL BIT(25) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED_COLL BIT(24) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED_ACT BIT(23) +#define AR933X_GPIO_FUNC_SPI_EN BIT(18) +#define AR933X_GPIO_FUNC_SPI_CS_EN2 BIT(14) +#define AR933X_GPIO_FUNC_SPI_CS_EN1 BIT(13) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED4_EN BIT(7) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED3_EN BIT(6) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED2_EN BIT(5) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED1_EN BIT(4) +#define AR933X_GPIO_FUNC_ETH_SWITCH_LED0_EN BIT(3) +#define AR933X_GPIO_FUNC_UART_RTS_CTS_EN BIT(2) +#define AR933X_GPIO_FUNC_UART_EN BIT(1) +#define AR933X_GPIO_FUNC_JTAG_DISABLE BIT(0) + +#define AR934X_GPIO_FUNC_CLK_OBS7_EN BIT(9) +#define AR934X_GPIO_FUNC_CLK_OBS6_EN BIT(8) +#define AR934X_GPIO_FUNC_CLK_OBS5_EN BIT(7) +#define AR934X_GPIO_FUNC_CLK_OBS4_EN BIT(6) +#define AR934X_GPIO_FUNC_CLK_OBS3_EN BIT(5) +#define AR934X_GPIO_FUNC_CLK_OBS2_EN BIT(4) +#define AR934X_GPIO_FUNC_CLK_OBS1_EN BIT(3) +#define AR934X_GPIO_FUNC_CLK_OBS0_EN BIT(2) +#define AR934X_GPIO_FUNC_JTAG_DISABLE BIT(1) + +#define AR934X_GPIO_OUT_GPIO 0 +#define AR934X_GPIO_OUT_SPI_CS1 7 +#define AR934X_GPIO_OUT_LED_LINK0 41 +#define AR934X_GPIO_OUT_LED_LINK1 42 +#define AR934X_GPIO_OUT_LED_LINK2 43 +#define AR934X_GPIO_OUT_LED_LINK3 44 +#define AR934X_GPIO_OUT_LED_LINK4 45 +#define AR934X_GPIO_OUT_EXT_LNA0 46 +#define AR934X_GPIO_OUT_EXT_LNA1 47 + +#define QCA955X_GPIO_FUNC_CLK_OBS7_EN BIT(9) +#define QCA955X_GPIO_FUNC_CLK_OBS6_EN BIT(8) +#define QCA955X_GPIO_FUNC_CLK_OBS5_EN BIT(7) +#define QCA955X_GPIO_FUNC_CLK_OBS4_EN BIT(6) +#define QCA955X_GPIO_FUNC_CLK_OBS3_EN BIT(5) +#define QCA955X_GPIO_FUNC_CLK_OBS2_EN BIT(4) +#define QCA955X_GPIO_FUNC_CLK_OBS1_EN BIT(3) +#define QCA955X_GPIO_FUNC_JTAG_DISABLE BIT(1) + +#define QCA955X_GPIO_OUT_GPIO 0 +#define QCA955X_MII_EXT_MDI 1 +#define QCA955X_SLIC_DATA_OUT 3 +#define QCA955X_SLIC_PCM_FS 4 +#define QCA955X_SLIC_PCM_CLK 5 +#define QCA955X_SPI_CLK 8 +#define QCA955X_SPI_CS_0 9 +#define QCA955X_SPI_CS_1 10 +#define QCA955X_SPI_CS_2 11 +#define QCA955X_SPI_MISO 12 +#define QCA955X_I2S_CLK 13 +#define QCA955X_I2S_WS 14 +#define QCA955X_I2S_SD 15 +#define QCA955X_I2S_MCK 16 +#define QCA955X_SPDIF_OUT 17 +#define QCA955X_UART1_TD 18 +#define QCA955X_UART1_RTS 19 +#define QCA955X_UART1_RD 20 +#define QCA955X_UART1_CTS 21 +#define QCA955X_UART0_SOUT 22 +#define QCA955X_SPDIF2_OUT 23 +#define QCA955X_LED_SGMII_SPEED0 24 +#define QCA955X_LED_SGMII_SPEED1 25 +#define QCA955X_LED_SGMII_DUPLEX 26 +#define QCA955X_LED_SGMII_LINK_UP 27 +#define QCA955X_SGMII_SPEED0_INVERT 28 +#define QCA955X_SGMII_SPEED1_INVERT 29 +#define QCA955X_SGMII_DUPLEX_INVERT 30 +#define QCA955X_SGMII_LINK_UP_INVERT 31 +#define QCA955X_GE1_MII_MDO 32 +#define QCA955X_GE1_MII_MDC 33 +#define QCA955X_SWCOM2 38 +#define QCA955X_SWCOM3 39 +#define QCA955X_MAC2_GPIO 40 +#define QCA955X_MAC3_GPIO 41 +#define QCA955X_ATT_LED 42 +#define QCA955X_PWR_LED 43 +#define QCA955X_TX_FRAME 44 +#define QCA955X_RX_CLEAR_EXTERNAL 45 +#define QCA955X_LED_NETWORK_EN 46 +#define QCA955X_LED_POWER_EN 47 +#define QCA955X_WMAC_GLUE_WOW 68 +#define QCA955X_RX_CLEAR_EXTENSION 70 +#define QCA955X_CP_NAND_CS1 73 +#define QCA955X_USB_SUSPEND 74 +#define QCA955X_ETH_TX_ERR 75 +#define QCA955X_DDR_DQ_OE 76 +#define QCA955X_CLKREQ_N_EP 77 +#define QCA955X_CLKREQ_N_RC 78 +#define QCA955X_CLK_OBS0 79 +#define QCA955X_CLK_OBS1 80 +#define QCA955X_CLK_OBS2 81 +#define QCA955X_CLK_OBS3 82 +#define QCA955X_CLK_OBS4 83 +#define QCA955X_CLK_OBS5 84 + +/* + * MII_CTRL block + */ +#define AR71XX_MII_REG_MII0_CTRL 0x00 +#define AR71XX_MII_REG_MII1_CTRL 0x04 + +#define AR71XX_MII_CTRL_IF_MASK 3 +#define AR71XX_MII_CTRL_SPEED_SHIFT 4 +#define AR71XX_MII_CTRL_SPEED_MASK 3 +#define AR71XX_MII_CTRL_SPEED_10 0 +#define AR71XX_MII_CTRL_SPEED_100 1 +#define AR71XX_MII_CTRL_SPEED_1000 2 + +#define AR71XX_MII0_CTRL_IF_GMII 0 +#define AR71XX_MII0_CTRL_IF_MII 1 +#define AR71XX_MII0_CTRL_IF_RGMII 2 +#define AR71XX_MII0_CTRL_IF_RMII 3 + +#define AR71XX_MII1_CTRL_IF_RGMII 0 +#define AR71XX_MII1_CTRL_IF_RMII 1 + +/* + * AR933X GMAC interface + */ +#define AR933X_GMAC_REG_ETH_CFG 0x00 + +#define AR933X_ETH_CFG_RGMII_GE0 BIT(0) +#define AR933X_ETH_CFG_MII_GE0 BIT(1) +#define AR933X_ETH_CFG_GMII_GE0 BIT(2) +#define AR933X_ETH_CFG_MII_GE0_MASTER BIT(3) +#define AR933X_ETH_CFG_MII_GE0_SLAVE BIT(4) +#define AR933X_ETH_CFG_MII_GE0_ERR_EN BIT(5) +#define AR933X_ETH_CFG_SW_PHY_SWAP BIT(7) +#define AR933X_ETH_CFG_SW_PHY_ADDR_SWAP BIT(8) +#define AR933X_ETH_CFG_RMII_GE0 BIT(9) +#define AR933X_ETH_CFG_RMII_GE0_SPD_10 0 +#define AR933X_ETH_CFG_RMII_GE0_SPD_100 BIT(10) + +/* + * AR934X GMAC Interface + */ +#define AR934X_GMAC_REG_ETH_CFG 0x00 + +#define AR934X_ETH_CFG_RGMII_GMAC0 BIT(0) +#define AR934X_ETH_CFG_MII_GMAC0 BIT(1) +#define AR934X_ETH_CFG_GMII_GMAC0 BIT(2) +#define AR934X_ETH_CFG_MII_GMAC0_MASTER BIT(3) +#define AR934X_ETH_CFG_MII_GMAC0_SLAVE BIT(4) +#define AR934X_ETH_CFG_MII_GMAC0_ERR_EN BIT(5) +#define AR934X_ETH_CFG_SW_ONLY_MODE BIT(6) +#define AR934X_ETH_CFG_SW_PHY_SWAP BIT(7) +#define AR934X_ETH_CFG_SW_APB_ACCESS BIT(9) +#define AR934X_ETH_CFG_RMII_GMAC0 BIT(10) +#define AR933X_ETH_CFG_MII_CNTL_SPEED BIT(11) +#define AR934X_ETH_CFG_RMII_GMAC0_MASTER BIT(12) +#define AR933X_ETH_CFG_SW_ACC_MSB_FIRST BIT(13) +#define AR934X_ETH_CFG_RXD_DELAY BIT(14) +#define AR934X_ETH_CFG_RXD_DELAY_MASK 0x3 +#define AR934X_ETH_CFG_RXD_DELAY_SHIFT 14 +#define AR934X_ETH_CFG_RDV_DELAY BIT(16) +#define AR934X_ETH_CFG_RDV_DELAY_MASK 0x3 +#define AR934X_ETH_CFG_RDV_DELAY_SHIFT 16 + +/* + * QCA953X GMAC Interface + */ +#define QCA953X_GMAC_REG_ETH_CFG 0x00 + +#define QCA953X_ETH_CFG_SW_ONLY_MODE BIT(6) +#define QCA953X_ETH_CFG_SW_PHY_SWAP BIT(7) +#define QCA953X_ETH_CFG_SW_APB_ACCESS BIT(9) +#define QCA953X_ETH_CFG_SW_ACC_MSB_FIRST BIT(13) + +/* + * QCA955X GMAC Interface + */ + +#define QCA955X_GMAC_REG_ETH_CFG 0x00 +#define QCA955X_GMAC_REG_SGMII_SERDES 0x18 + +#define QCA955X_ETH_CFG_RGMII_EN BIT(0) +#define QCA955X_ETH_CFG_MII_GE0 BIT(1) +#define QCA955X_ETH_CFG_GMII_GE0 BIT(2) +#define QCA955X_ETH_CFG_MII_GE0_MASTER BIT(3) +#define QCA955X_ETH_CFG_MII_GE0_SLAVE BIT(4) +#define QCA955X_ETH_CFG_GE0_ERR_EN BIT(5) +#define QCA955X_ETH_CFG_GE0_SGMII BIT(6) +#define QCA955X_ETH_CFG_RMII_GE0 BIT(10) +#define QCA955X_ETH_CFG_MII_CNTL_SPEED BIT(11) +#define QCA955X_ETH_CFG_RMII_GE0_MASTER BIT(12) +#define QCA955X_ETH_CFG_RXD_DELAY_MASK 0x3 +#define QCA955X_ETH_CFG_RXD_DELAY_SHIFT 14 +#define QCA955X_ETH_CFG_RDV_DELAY BIT(16) +#define QCA955X_ETH_CFG_RDV_DELAY_MASK 0x3 +#define QCA955X_ETH_CFG_RDV_DELAY_SHIFT 16 +#define QCA955X_ETH_CFG_TXD_DELAY_MASK 0x3 +#define QCA955X_ETH_CFG_TXD_DELAY_SHIFT 18 +#define QCA955X_ETH_CFG_TXE_DELAY_MASK 0x3 +#define QCA955X_ETH_CFG_TXE_DELAY_SHIFT 20 + +#define QCA955X_SGMII_SERDES_LOCK_DETECT_STATUS BIT(15) +#define QCA955X_SGMII_SERDES_RES_CALIBRATION_SHIFT 23 +#define QCA955X_SGMII_SERDES_RES_CALIBRATION_MASK 0xf +/* + * QCA956X GMAC Interface + */ + +#define QCA956X_GMAC_REG_ETH_CFG 0x00 +#define QCA956X_GMAC_REG_SGMII_RESET 0x14 +#define QCA956X_GMAC_REG_SGMII_SERDES 0x18 +#define QCA956X_GMAC_REG_MR_AN_CONTROL 0x1c +#define QCA956X_GMAC_REG_SGMII_CONFIG 0x34 +#define QCA956X_GMAC_REG_SGMII_DEBUG 0x58 + +#define QCA956X_ETH_CFG_RGMII_EN BIT(0) +#define QCA956X_ETH_CFG_GE0_SGMII BIT(6) +#define QCA956X_ETH_CFG_SW_ONLY_MODE BIT(7) +#define QCA956X_ETH_CFG_SW_PHY_SWAP BIT(8) +#define QCA956X_ETH_CFG_SW_PHY_ADDR_SWAP BIT(9) +#define QCA956X_ETH_CFG_SW_APB_ACCESS BIT(10) +#define QCA956X_ETH_CFG_SW_ACC_MSB_FIRST BIT(13) +#define QCA956X_ETH_CFG_RXD_DELAY_MASK 0x3 +#define QCA956X_ETH_CFG_RXD_DELAY_SHIFT 14 +#define QCA956X_ETH_CFG_RDV_DELAY_MASK 0x3 +#define QCA956X_ETH_CFG_RDV_DELAY_SHIFT 16 + +#define QCA956X_SGMII_RESET_RX_CLK_N_RESET 0x0 +#define QCA956X_SGMII_RESET_RX_CLK_N BIT(0) +#define QCA956X_SGMII_RESET_TX_CLK_N BIT(1) +#define QCA956X_SGMII_RESET_RX_125M_N BIT(2) +#define QCA956X_SGMII_RESET_TX_125M_N BIT(3) +#define QCA956X_SGMII_RESET_HW_RX_125M_N BIT(4) + +#define QCA956X_SGMII_SERDES_CDR_BW_MASK 0x3 +#define QCA956X_SGMII_SERDES_CDR_BW_SHIFT 1 +#define QCA956X_SGMII_SERDES_TX_DR_CTRL_MASK 0x7 +#define QCA956X_SGMII_SERDES_TX_DR_CTRL_SHIFT 4 +#define QCA956X_SGMII_SERDES_PLL_BW BIT(8) +#define QCA956X_SGMII_SERDES_VCO_FAST BIT(9) +#define QCA956X_SGMII_SERDES_VCO_SLOW BIT(10) +#define QCA956X_SGMII_SERDES_LOCK_DETECT_STATUS BIT(15) +#define QCA956X_SGMII_SERDES_EN_SIGNAL_DETECT BIT(16) +#define QCA956X_SGMII_SERDES_FIBER_SDO BIT(17) +#define QCA956X_SGMII_SERDES_RES_CALIBRATION_SHIFT 23 +#define QCA956X_SGMII_SERDES_RES_CALIBRATION_MASK 0xf +#define QCA956X_SGMII_SERDES_VCO_REG_SHIFT 27 +#define QCA956X_SGMII_SERDES_VCO_REG_MASK 0xf + +#define QCA956X_MR_AN_CONTROL_AN_ENABLE BIT(12) +#define QCA956X_MR_AN_CONTROL_PHY_RESET BIT(15) + +#define QCA956X_SGMII_CONFIG_MODE_CTRL_SHIFT 0 +#define QCA956X_SGMII_CONFIG_MODE_CTRL_MASK 0x7 + #endif /* __ASM_MACH_AR71XX_REGS_H */ diff --git a/arch/mips/include/asm/mach-ath79/ath79.h b/arch/mips/include/asm/mach-ath79/ath79.h index 441faa92c3cd..73dcd63b8243 100644 --- a/arch/mips/include/asm/mach-ath79/ath79.h +++ b/arch/mips/include/asm/mach-ath79/ath79.h @@ -32,8 +32,11 @@ enum ath79_soc_type { ATH79_SOC_AR9341, ATH79_SOC_AR9342, ATH79_SOC_AR9344, + ATH79_SOC_QCA9533, ATH79_SOC_QCA9556, ATH79_SOC_QCA9558, + ATH79_SOC_TP9343, + ATH79_SOC_QCA956X, }; extern enum ath79_soc_type ath79_soc; @@ -100,6 +103,16 @@ static inline int soc_is_ar934x(void) return soc_is_ar9341() || soc_is_ar9342() || soc_is_ar9344(); } +static inline int soc_is_qca9533(void) +{ + return ath79_soc == ATH79_SOC_QCA9533; +} + +static inline int soc_is_qca953x(void) +{ + return soc_is_qca9533(); +} + static inline int soc_is_qca9556(void) { return ath79_soc == ATH79_SOC_QCA9556; @@ -115,6 +128,26 @@ static inline int soc_is_qca955x(void) return soc_is_qca9556() || soc_is_qca9558(); } +static inline int soc_is_tp9343(void) +{ + return ath79_soc == ATH79_SOC_TP9343; +} + +static inline int soc_is_qca9561(void) +{ + return ath79_soc == ATH79_SOC_QCA956X; +} + +static inline int soc_is_qca9563(void) +{ + return ath79_soc == ATH79_SOC_QCA956X; +} + +static inline int soc_is_qca956x(void) +{ + return soc_is_qca9561() || soc_is_qca9563(); +} + void ath79_ddr_wb_flush(unsigned int reg); void ath79_ddr_set_pci_windows(void); @@ -134,6 +167,7 @@ static inline u32 ath79_pll_rr(unsigned reg) static inline void ath79_reset_wr(unsigned reg, u32 val) { __raw_writel(val, ath79_reset_base + reg); + (void) __raw_readl(ath79_reset_base + reg); /* flush */ } static inline u32 ath79_reset_rr(unsigned reg) diff --git a/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h index 0089a740e5ae..026ad90c8ac0 100644 --- a/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h @@ -36,6 +36,7 @@ #define cpu_has_mdmx 0 #define cpu_has_mips3d 0 #define cpu_has_smartmips 0 +#define cpu_has_rixi 0 #define cpu_has_mips32r1 1 #define cpu_has_mips32r2 1 @@ -43,6 +44,7 @@ #define cpu_has_mips64r2 0 #define cpu_has_mipsmt 0 +#define cpu_has_userlocal 0 #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 @@ -51,5 +53,9 @@ #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 +#define cpu_has_vtag_icache 0 +#define cpu_has_dc_aliases 1 +#define cpu_has_ic_fills_f_dc 0 +#define cpu_has_pindexed_dcache 0 #endif /* __ASM_MACH_ATH79_CPU_FEATURE_OVERRIDES_H */ diff --git a/arch/mips/include/asm/mach-bmips/dma-coherence.h b/arch/mips/include/asm/mach-bmips/dma-coherence.h deleted file mode 100644 index d29781f02285..000000000000 --- a/arch/mips/include/asm/mach-bmips/dma-coherence.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * Copyright (C) 2009 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#ifndef __ASM_MACH_BMIPS_DMA_COHERENCE_H -#define __ASM_MACH_BMIPS_DMA_COHERENCE_H - -#include <asm/bmips.h> -#include <asm/cpu-type.h> -#include <asm/cpu.h> - -struct device; - -extern dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, size_t size); -extern dma_addr_t plat_map_dma_mem_page(struct device *dev, struct page *page); -extern unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr); - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline int plat_device_is_coherent(struct device *dev) -{ - return 0; -} - -#define plat_post_dma_flush bmips_post_dma_flush - -#endif /* __ASM_MACH_BMIPS_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h b/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h deleted file mode 100644 index 6eb1ee548b11..000000000000 --- a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * - * - * Similar to mach-generic/dma-coherence.h except - * plat_device_is_coherent hard coded to return 1. - * - */ -#ifndef __ASM_MACH_CAVIUM_OCTEON_DMA_COHERENCE_H -#define __ASM_MACH_CAVIUM_OCTEON_DMA_COHERENCE_H - -#include <linux/bug.h> - -struct device; - -extern void octeon_pci_dma_init(void); - -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, - size_t size) -{ - BUG(); - return 0; -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ - BUG(); - return 0; -} - -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ - BUG(); - return 0; -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ - BUG(); -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - BUG(); - return 0; -} - -static inline int plat_device_is_coherent(struct device *dev) -{ - return 1; -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - if (!dev->dma_mask) - return false; - - return addr + size - 1 <= *dev->dma_mask; -} - -dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr); -phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr); - -struct dma_map_ops; -extern const struct dma_map_ops *octeon_pci_dma_map_ops; -extern char *octeon_swiotlb; - -#endif /* __ASM_MACH_CAVIUM_OCTEON_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-generic/dma-coherence.h b/arch/mips/include/asm/mach-generic/dma-coherence.h deleted file mode 100644 index 8ad7a40ca786..000000000000 --- a/arch/mips/include/asm/mach-generic/dma-coherence.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * - */ -#ifndef __ASM_MACH_GENERIC_DMA_COHERENCE_H -#define __ASM_MACH_GENERIC_DMA_COHERENCE_H - -struct device; - -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, - size_t size) -{ - return virt_to_phys(addr); -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ - return page_to_phys(page); -} - -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ - return dma_addr; -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline int plat_device_is_coherent(struct device *dev) -{ -#ifdef CONFIG_DMA_PERDEV_COHERENT - return dev->archdata.dma_coherent; -#else - switch (coherentio) { - default: - case IO_COHERENCE_DEFAULT: - return hw_coherentio; - case IO_COHERENCE_ENABLED: - return 1; - case IO_COHERENCE_DISABLED: - return 0; - } -#endif -} - -#ifndef plat_post_dma_flush -static inline void plat_post_dma_flush(struct device *dev) -{ -} -#endif - -#endif /* __ASM_MACH_GENERIC_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-generic/kmalloc.h b/arch/mips/include/asm/mach-generic/kmalloc.h index 74207c7bd00d..649a98338886 100644 --- a/arch/mips/include/asm/mach-generic/kmalloc.h +++ b/arch/mips/include/asm/mach-generic/kmalloc.h @@ -2,8 +2,7 @@ #ifndef __ASM_MACH_GENERIC_KMALLOC_H #define __ASM_MACH_GENERIC_KMALLOC_H - -#ifndef CONFIG_DMA_COHERENT +#ifdef CONFIG_DMA_NONCOHERENT /* * Total overkill for most systems but need as a safe default. * Set this one if any device in the system might do non-coherent DMA. diff --git a/arch/mips/include/asm/mach-generic/spaces.h b/arch/mips/include/asm/mach-generic/spaces.h index 952b0fdfda0e..ee5ebe98f6cf 100644 --- a/arch/mips/include/asm/mach-generic/spaces.h +++ b/arch/mips/include/asm/mach-generic/spaces.h @@ -17,9 +17,13 @@ /* * This gives the physical RAM offset. */ -#ifndef PHYS_OFFSET -#define PHYS_OFFSET _AC(0, UL) -#endif +#ifndef __ASSEMBLY__ +# if defined(CONFIG_MIPS_AUTO_PFN_OFFSET) +# define PHYS_OFFSET ((unsigned long)PFN_PHYS(ARCH_PFN_OFFSET)) +# elif !defined(PHYS_OFFSET) +# define PHYS_OFFSET _AC(0, UL) +# endif +#endif /* __ASSEMBLY__ */ #ifdef CONFIG_32BIT #ifdef CONFIG_KVM_GUEST diff --git a/arch/mips/include/asm/mach-ip27/dma-coherence.h b/arch/mips/include/asm/mach-ip27/dma-coherence.h deleted file mode 100644 index 04d862020ac9..000000000000 --- a/arch/mips/include/asm/mach-ip27/dma-coherence.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * - */ -#ifndef __ASM_MACH_IP27_DMA_COHERENCE_H -#define __ASM_MACH_IP27_DMA_COHERENCE_H - -#include <asm/pci/bridge.h> - -#define pdev_to_baddr(pdev, addr) \ - (BRIDGE_CONTROLLER(pdev->bus)->baddr + (addr)) -#define dev_to_baddr(dev, addr) \ - pdev_to_baddr(to_pci_dev(dev), (addr)) - -struct device; - -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, - size_t size) -{ - dma_addr_t pa = dev_to_baddr(dev, virt_to_phys(addr)); - - return pa; -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ - dma_addr_t pa = dev_to_baddr(dev, page_to_phys(page)); - - return pa; -} - -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ - return dma_addr & ~(0xffUL << 56); -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -static inline int plat_device_is_coherent(struct device *dev) -{ - return 1; /* IP27 non-coherent mode is unsupported */ -} - -#endif /* __ASM_MACH_IP27_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-ip32/dma-coherence.h b/arch/mips/include/asm/mach-ip32/dma-coherence.h deleted file mode 100644 index 7bdf212587a0..000000000000 --- a/arch/mips/include/asm/mach-ip32/dma-coherence.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - * - */ -#ifndef __ASM_MACH_IP32_DMA_COHERENCE_H -#define __ASM_MACH_IP32_DMA_COHERENCE_H - -#include <asm/ip32/crime.h> - -struct device; - -/* - * Few notes. - * 1. CPU sees memory as two chunks: 0-256M@0x0, and the rest @0x40000000+256M - * 2. PCI sees memory as one big chunk @0x0 (or we could use 0x40000000 for - * native-endian) - * 3. All other devices see memory as one big chunk at 0x40000000 - * 4. Non-PCI devices will pass NULL as struct device* - * - * Thus we translate differently, depending on device. - */ - -#define RAM_OFFSET_MASK 0x3fffffffUL - -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, - size_t size) -{ - dma_addr_t pa = virt_to_phys(addr) & RAM_OFFSET_MASK; - - if (dev == NULL) - pa += CRIME_HI_MEM_BASE; - - return pa; -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ - dma_addr_t pa; - - pa = page_to_phys(page) & RAM_OFFSET_MASK; - - if (dev == NULL) - pa += CRIME_HI_MEM_BASE; - - return pa; -} - -/* This is almost certainly wrong but it's what dma-ip32.c used to use */ -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ - unsigned long addr = dma_addr & RAM_OFFSET_MASK; - - if (dma_addr >= 256*1024*1024) - addr += CRIME_HI_MEM_BASE; - - return addr; -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -static inline int plat_device_is_coherent(struct device *dev) -{ - return 0; /* IP32 is non-coherent */ -} - -#endif /* __ASM_MACH_IP32_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-jazz/dma-coherence.h b/arch/mips/include/asm/mach-jazz/dma-coherence.h deleted file mode 100644 index dc347c25c343..000000000000 --- a/arch/mips/include/asm/mach-jazz/dma-coherence.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> - */ -#ifndef __ASM_MACH_JAZZ_DMA_COHERENCE_H -#define __ASM_MACH_JAZZ_DMA_COHERENCE_H - -#include <asm/jazzdma.h> - -struct device; - -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, size_t size) -{ - return vdma_alloc(virt_to_phys(addr), size); -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ - return vdma_alloc(page_to_phys(page), PAGE_SIZE); -} - -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ - return vdma_log2phys(dma_addr); -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ - vdma_free(dma_addr); -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -static inline int plat_device_is_coherent(struct device *dev) -{ - return 0; -} - -#endif /* __ASM_MACH_JAZZ_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-loongson64/dma-coherence.h b/arch/mips/include/asm/mach-loongson64/dma-coherence.h deleted file mode 100644 index 64fc44dec0a8..000000000000 --- a/arch/mips/include/asm/mach-loongson64/dma-coherence.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006, 07 Ralf Baechle <ralf@linux-mips.org> - * Copyright (C) 2007 Lemote, Inc. & Institute of Computing Technology - * Author: Fuxin Zhang, zhangfx@lemote.com - * - */ -#ifndef __ASM_MACH_LOONGSON64_DMA_COHERENCE_H -#define __ASM_MACH_LOONGSON64_DMA_COHERENCE_H - -#ifdef CONFIG_SWIOTLB -#include <linux/swiotlb.h> -#endif - -struct device; - -static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) -{ - if (!dev->dma_mask) - return false; - - return addr + size - 1 <= *dev->dma_mask; -} - -extern dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr); -extern phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr); -static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, - size_t size) -{ -#ifdef CONFIG_CPU_LOONGSON3 - return __phys_to_dma(dev, virt_to_phys(addr)); -#else - return virt_to_phys(addr) | 0x80000000; -#endif -} - -static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, - struct page *page) -{ -#ifdef CONFIG_CPU_LOONGSON3 - return __phys_to_dma(dev, page_to_phys(page)); -#else - return page_to_phys(page) | 0x80000000; -#endif -} - -static inline unsigned long plat_dma_addr_to_phys(struct device *dev, - dma_addr_t dma_addr) -{ -#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT) - return __dma_to_phys(dev, dma_addr); -#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT) - return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff); -#else - return dma_addr & 0x7fffffff; -#endif -} - -static inline void plat_unmap_dma_mem(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction) -{ -} - -static inline int plat_dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if (mask < DMA_BIT_MASK(24)) - return 0; - - return 1; -} - -static inline int plat_device_is_coherent(struct device *dev) -{ -#ifdef CONFIG_DMA_NONCOHERENT - return 0; -#else - return 1; -#endif /* CONFIG_DMA_NONCOHERENT */ -} - -static inline void plat_post_dma_flush(struct device *dev) -{ -} - -#endif /* __ASM_MACH_LOONGSON64_DMA_COHERENCE_H */ diff --git a/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h b/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h index 8393bc548987..312739117bb0 100644 --- a/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h +++ b/arch/mips/include/asm/mach-loongson64/kernel-entry-init.h @@ -19,18 +19,18 @@ .set push .set mips64 /* Set LPA on LOONGSON3 config3 */ - mfc0 t0, $16, 3 + mfc0 t0, CP0_CONFIG3 or t0, (0x1 << 7) - mtc0 t0, $16, 3 + mtc0 t0, CP0_CONFIG3 /* Set ELPA on LOONGSON3 pagegrain */ - mfc0 t0, $5, 1 + mfc0 t0, CP0_PAGEGRAIN or t0, (0x1 << 29) - mtc0 t0, $5, 1 + mtc0 t0, CP0_PAGEGRAIN #ifdef CONFIG_LOONGSON3_ENHANCEMENT /* Enable STFill Buffer */ - mfc0 t0, $16, 6 + mfc0 t0, CP0_CONFIG6 or t0, 0x100 - mtc0 t0, $16, 6 + mtc0 t0, CP0_CONFIG6 #endif _ehb .set pop @@ -45,18 +45,18 @@ .set push .set mips64 /* Set LPA on LOONGSON3 config3 */ - mfc0 t0, $16, 3 + mfc0 t0, CP0_CONFIG3 or t0, (0x1 << 7) - mtc0 t0, $16, 3 + mtc0 t0, CP0_CONFIG3 /* Set ELPA on LOONGSON3 pagegrain */ - mfc0 t0, $5, 1 + mfc0 t0, CP0_PAGEGRAIN or t0, (0x1 << 29) - mtc0 t0, $5, 1 + mtc0 t0, CP0_PAGEGRAIN #ifdef CONFIG_LOONGSON3_ENHANCEMENT /* Enable STFill Buffer */ - mfc0 t0, $16, 6 + mfc0 t0, CP0_CONFIG6 or t0, 0x100 - mtc0 t0, $16, 6 + mtc0 t0, CP0_CONFIG6 #endif _ehb .set pop diff --git a/arch/mips/include/asm/mach-pic32/spaces.h b/arch/mips/include/asm/mach-pic32/spaces.h index 046a0a9aa8b3..a1b9783b76ea 100644 --- a/arch/mips/include/asm/mach-pic32/spaces.h +++ b/arch/mips/include/asm/mach-pic32/spaces.h @@ -16,7 +16,6 @@ #ifdef CONFIG_PIC32MZDA #define PHYS_OFFSET _AC(0x08000000, UL) -#define UNCAC_BASE _AC(0xa8000000, UL) #endif #include <asm/mach-generic/spaces.h> diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 0bc270806ec5..01df9ad62fb8 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -16,6 +16,7 @@ #include <linux/linkage.h> #include <linux/types.h> #include <asm/hazards.h> +#include <asm/isa-rev.h> #include <asm/war.h> /* @@ -51,6 +52,7 @@ #define CP0_GLOBALNUMBER $3, 1 #define CP0_CONTEXT $4 #define CP0_PAGEMASK $5 +#define CP0_PAGEGRAIN $5, 1 #define CP0_SEGCTL0 $5, 2 #define CP0_SEGCTL1 $5, 3 #define CP0_SEGCTL2 $5, 4 @@ -77,6 +79,7 @@ #define CP0_CONFIG $16 #define CP0_CONFIG3 $16, 3 #define CP0_CONFIG5 $16, 5 +#define CP0_CONFIG6 $16, 6 #define CP0_LLADDR $17 #define CP0_WATCHLO $18 #define CP0_WATCHHI $19 @@ -681,8 +684,6 @@ #define MIPS_CONF7_WII (_ULCAST_(1) << 31) #define MIPS_CONF7_RPS (_ULCAST_(1) << 2) -/* ExternalSync */ -#define MIPS_CONF7_ES (_ULCAST_(1) << 8) #define MIPS_CONF7_IAR (_ULCAST_(1) << 10) #define MIPS_CONF7_AR (_ULCAST_(1) << 16) @@ -1483,32 +1484,38 @@ do { \ #define __write_64bit_c0_split(source, sel, val) \ do { \ - unsigned long long __tmp; \ + unsigned long long __tmp = (val); \ unsigned long __flags; \ \ local_irq_save(__flags); \ - if (sel == 0) \ + if (MIPS_ISA_REV >= 2) \ + __asm__ __volatile__( \ + ".set\tpush\n\t" \ + ".set\t" MIPS_ISA_LEVEL "\n\t" \ + "dins\t%L0, %M0, 32, 32\n\t" \ + "dmtc0\t%L0, " #source ", " #sel "\n\t" \ + ".set\tpop" \ + : "+r" (__tmp)); \ + else if (sel == 0) \ __asm__ __volatile__( \ ".set\tmips64\n\t" \ - "dsll\t%L0, %L1, 32\n\t" \ + "dsll\t%L0, %L0, 32\n\t" \ "dsrl\t%L0, %L0, 32\n\t" \ - "dsll\t%M0, %M1, 32\n\t" \ + "dsll\t%M0, %M0, 32\n\t" \ "or\t%L0, %L0, %M0\n\t" \ "dmtc0\t%L0, " #source "\n\t" \ ".set\tmips0" \ - : "=&r,r" (__tmp) \ - : "r,0" (val)); \ + : "+r" (__tmp)); \ else \ __asm__ __volatile__( \ ".set\tmips64\n\t" \ - "dsll\t%L0, %L1, 32\n\t" \ + "dsll\t%L0, %L0, 32\n\t" \ "dsrl\t%L0, %L0, 32\n\t" \ - "dsll\t%M0, %M1, 32\n\t" \ + "dsll\t%M0, %M0, 32\n\t" \ "or\t%L0, %L0, %M0\n\t" \ "dmtc0\t%L0, " #source ", " #sel "\n\t" \ ".set\tmips0" \ - : "=&r,r" (__tmp) \ - : "r,0" (val)); \ + : "+r" (__tmp)); \ local_irq_restore(__flags); \ } while (0) @@ -2767,7 +2774,6 @@ __BUILD_SET_C0(status) __BUILD_SET_C0(cause) __BUILD_SET_C0(config) __BUILD_SET_C0(config5) -__BUILD_SET_C0(config7) __BUILD_SET_C0(intcontrol) __BUILD_SET_C0(intctl) __BUILD_SET_C0(srsmap) diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h index da2004cef2d5..b509371a6b0c 100644 --- a/arch/mips/include/asm/mmu_context.h +++ b/arch/mips/include/asm/mmu_context.h @@ -126,8 +126,6 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) for_each_possible_cpu(i) cpu_context(i, mm) = 0; - atomic_set(&mm->context.fp_mode_switching, 0); - mm->context.bd_emupage_allocmap = NULL; spin_lock_init(&mm->context.bd_emupage_lock); init_waitqueue_head(&mm->context.bd_emupage_queue); diff --git a/arch/mips/include/asm/netlogic/xlr/fmn.h b/arch/mips/include/asm/netlogic/xlr/fmn.h index 5604db3d1836..d79c68fa78d9 100644 --- a/arch/mips/include/asm/netlogic/xlr/fmn.h +++ b/arch/mips/include/asm/netlogic/xlr/fmn.h @@ -301,8 +301,6 @@ static inline int nlm_fmn_send(unsigned int size, unsigned int code, for (i = 0; i < 8; i++) { nlm_msgsnd(dest); status = nlm_read_c2_status0(); - if ((status & 0x2) == 1) - pr_info("Send pending fail!\n"); if ((status & 0x4) == 0) return 0; } diff --git a/arch/mips/include/asm/octeon/cvmx-asxx-defs.h b/arch/mips/include/asm/octeon/cvmx-asxx-defs.h index a1e21a3854cf..1eef155979f3 100644 --- a/arch/mips/include/asm/octeon/cvmx-asxx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-asxx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -55,6 +55,8 @@ #define CVMX_ASXX_TX_HI_WATERX(offset, block_id) (CVMX_ADD_IO_SEG(0x00011800B0000080ull) + (((offset) & 3) + ((block_id) & 1) * 0x1000000ull) * 8) #define CVMX_ASXX_TX_PRT_EN(block_id) (CVMX_ADD_IO_SEG(0x00011800B0000008ull) + ((block_id) & 1) * 0x8000000ull) +void __cvmx_interrupt_asxx_enable(int block); + union cvmx_asxx_gmii_rx_clk_set { uint64_t u64; struct cvmx_asxx_gmii_rx_clk_set_s { diff --git a/arch/mips/include/asm/octeon/cvmx-ciu-defs.h b/arch/mips/include/asm/octeon/cvmx-ciu-defs.h index 6e61792d9248..1d18be8cdddc 100644 --- a/arch/mips/include/asm/octeon/cvmx-ciu-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-ciu-defs.h @@ -1,10014 +1,176 @@ -/***********************license start*************** - * Author: Cavium Networks +/* SPDX-License-Identifier: GPL-2.0 */ +/* Octeon CIU definitions * - * Contact: support@caviumnetworks.com - * This file is part of the OCTEON SDK - * - * Copyright (c) 2003-2012 Cavium Networks - * - * This file is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, Version 2, as - * published by the Free Software Foundation. - * - * This file is distributed in the hope that it will be useful, but - * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or - * NONINFRINGEMENT. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License - * along with this file; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * or visit http://www.gnu.org/licenses/. - * - * This file may also be available under a different license from Cavium. - * Contact Cavium Networks for more information - ***********************license end**************************************/ + * Copyright (C) 2003-2018 Cavium, Inc. + */ #ifndef __CVMX_CIU_DEFS_H__ #define __CVMX_CIU_DEFS_H__ -#define CVMX_CIU_BIST (CVMX_ADD_IO_SEG(0x0001070000000730ull)) -#define CVMX_CIU_BLOCK_INT (CVMX_ADD_IO_SEG(0x00010700000007C0ull)) -#define CVMX_CIU_DINT (CVMX_ADD_IO_SEG(0x0001070000000720ull)) -#define CVMX_CIU_EN2_IOX_INT(offset) (CVMX_ADD_IO_SEG(0x000107000000A600ull) + ((offset) & 1) * 8) -#define CVMX_CIU_EN2_IOX_INT_W1C(offset) (CVMX_ADD_IO_SEG(0x000107000000CE00ull) + ((offset) & 1) * 8) -#define CVMX_CIU_EN2_IOX_INT_W1S(offset) (CVMX_ADD_IO_SEG(0x000107000000AE00ull) + ((offset) & 1) * 8) -#define CVMX_CIU_EN2_PPX_IP2(offset) (CVMX_ADD_IO_SEG(0x000107000000A000ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP2_W1C(offset) (CVMX_ADD_IO_SEG(0x000107000000C800ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP2_W1S(offset) (CVMX_ADD_IO_SEG(0x000107000000A800ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP3(offset) (CVMX_ADD_IO_SEG(0x000107000000A200ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP3_W1C(offset) (CVMX_ADD_IO_SEG(0x000107000000CA00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP3_W1S(offset) (CVMX_ADD_IO_SEG(0x000107000000AA00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP4(offset) (CVMX_ADD_IO_SEG(0x000107000000A400ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP4_W1C(offset) (CVMX_ADD_IO_SEG(0x000107000000CC00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_EN2_PPX_IP4_W1S(offset) (CVMX_ADD_IO_SEG(0x000107000000AC00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_FUSE (CVMX_ADD_IO_SEG(0x0001070000000728ull)) -#define CVMX_CIU_GSTOP (CVMX_ADD_IO_SEG(0x0001070000000710ull)) -#define CVMX_CIU_INT33_SUM0 (CVMX_ADD_IO_SEG(0x0001070000000110ull)) -#define CVMX_CIU_INTX_EN0(offset) (CVMX_ADD_IO_SEG(0x0001070000000200ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN0_W1C(offset) (CVMX_ADD_IO_SEG(0x0001070000002200ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN0_W1S(offset) (CVMX_ADD_IO_SEG(0x0001070000006200ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN1(offset) (CVMX_ADD_IO_SEG(0x0001070000000208ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN1_W1C(offset) (CVMX_ADD_IO_SEG(0x0001070000002208ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN1_W1S(offset) (CVMX_ADD_IO_SEG(0x0001070000006208ull) + ((offset) & 63) * 16) -#define CVMX_CIU_INTX_EN4_0(offset) (CVMX_ADD_IO_SEG(0x0001070000000C80ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_EN4_0_W1C(offset) (CVMX_ADD_IO_SEG(0x0001070000002C80ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_EN4_0_W1S(offset) (CVMX_ADD_IO_SEG(0x0001070000006C80ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_EN4_1(offset) (CVMX_ADD_IO_SEG(0x0001070000000C88ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_EN4_1_W1C(offset) (CVMX_ADD_IO_SEG(0x0001070000002C88ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_EN4_1_W1S(offset) (CVMX_ADD_IO_SEG(0x0001070000006C88ull) + ((offset) & 15) * 16) -#define CVMX_CIU_INTX_SUM0(offset) (CVMX_ADD_IO_SEG(0x0001070000000000ull) + ((offset) & 63) * 8) -#define CVMX_CIU_INTX_SUM4(offset) (CVMX_ADD_IO_SEG(0x0001070000000C00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_INT_DBG_SEL (CVMX_ADD_IO_SEG(0x00010700000007D0ull)) -#define CVMX_CIU_INT_SUM1 (CVMX_ADD_IO_SEG(0x0001070000000108ull)) -static inline uint64_t CVMX_CIU_MBOX_CLRX(unsigned long offset) +#include <asm/bitfield.h> + +#define CVMX_CIU_ADDR(addr, coreid, coremask, offset) \ + (CVMX_ADD_IO_SEG(0x0001070000000000ull + addr##ull) + \ + (((coreid) & (coremask)) * offset)) + +#define CVMX_CIU_EN2_PPX_IP4(c) CVMX_CIU_ADDR(0xA400, c, 0x0F, 8) +#define CVMX_CIU_EN2_PPX_IP4_W1C(c) CVMX_CIU_ADDR(0xCC00, c, 0x0F, 8) +#define CVMX_CIU_EN2_PPX_IP4_W1S(c) CVMX_CIU_ADDR(0xAC00, c, 0x0F, 8) +#define CVMX_CIU_FUSE CVMX_CIU_ADDR(0x0728, 0, 0x00, 0) +#define CVMX_CIU_INT_SUM1 CVMX_CIU_ADDR(0x0108, 0, 0x00, 0) +#define CVMX_CIU_INTX_EN0(c) CVMX_CIU_ADDR(0x0200, c, 0x3F, 16) +#define CVMX_CIU_INTX_EN0_W1C(c) CVMX_CIU_ADDR(0x2200, c, 0x3F, 16) +#define CVMX_CIU_INTX_EN0_W1S(c) CVMX_CIU_ADDR(0x6200, c, 0x3F, 16) +#define CVMX_CIU_INTX_EN1(c) CVMX_CIU_ADDR(0x0208, c, 0x3F, 16) +#define CVMX_CIU_INTX_EN1_W1C(c) CVMX_CIU_ADDR(0x2208, c, 0x3F, 16) +#define CVMX_CIU_INTX_EN1_W1S(c) CVMX_CIU_ADDR(0x6208, c, 0x3F, 16) +#define CVMX_CIU_INTX_SUM0(c) CVMX_CIU_ADDR(0x0000, c, 0x3F, 8) +#define CVMX_CIU_NMI CVMX_CIU_ADDR(0x0718, 0, 0x00, 0) +#define CVMX_CIU_PCI_INTA CVMX_CIU_ADDR(0x0750, 0, 0x00, 0) +#define CVMX_CIU_PP_BIST_STAT CVMX_CIU_ADDR(0x07E0, 0, 0x00, 0) +#define CVMX_CIU_PP_DBG CVMX_CIU_ADDR(0x0708, 0, 0x00, 0) +#define CVMX_CIU_PP_RST CVMX_CIU_ADDR(0x0700, 0, 0x00, 0) +#define CVMX_CIU_QLM0 CVMX_CIU_ADDR(0x0780, 0, 0x00, 0) +#define CVMX_CIU_QLM1 CVMX_CIU_ADDR(0x0788, 0, 0x00, 0) +#define CVMX_CIU_QLM_JTGC CVMX_CIU_ADDR(0x0768, 0, 0x00, 0) +#define CVMX_CIU_QLM_JTGD CVMX_CIU_ADDR(0x0770, 0, 0x00, 0) +#define CVMX_CIU_SOFT_BIST CVMX_CIU_ADDR(0x0738, 0, 0x00, 0) +#define CVMX_CIU_SOFT_PRST1 CVMX_CIU_ADDR(0x0758, 0, 0x00, 0) +#define CVMX_CIU_SOFT_PRST CVMX_CIU_ADDR(0x0748, 0, 0x00, 0) +#define CVMX_CIU_SOFT_RST CVMX_CIU_ADDR(0x0740, 0, 0x00, 0) +#define CVMX_CIU_SUM2_PPX_IP4(c) CVMX_CIU_ADDR(0x8C00, c, 0x0F, 8) +#define CVMX_CIU_TIM_MULTI_CAST CVMX_CIU_ADDR(0xC200, 0, 0x00, 0) +#define CVMX_CIU_TIMX(c) CVMX_CIU_ADDR(0x0480, c, 0x0F, 8) + +static inline uint64_t CVMX_CIU_MBOX_CLRX(unsigned int coreid) { - switch (cvmx_get_octeon_family()) { - case OCTEON_CN30XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN52XX & OCTEON_FAMILY_MASK: - case OCTEON_CNF71XX & OCTEON_FAMILY_MASK: - case OCTEON_CN61XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN31XX & OCTEON_FAMILY_MASK: - case OCTEON_CN50XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN38XX & OCTEON_FAMILY_MASK: - case OCTEON_CN58XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN56XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN66XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN63XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; - case OCTEON_CN68XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070100100600ull) + (offset) * 8; - } - return CVMX_ADD_IO_SEG(0x0001070000000680ull) + (offset) * 8; + if (cvmx_get_octeon_family() == (OCTEON_CN68XX & OCTEON_FAMILY_MASK)) + return CVMX_CIU_ADDR(0x100100600, coreid, 0x0F, 8); + else + return CVMX_CIU_ADDR(0x000000680, coreid, 0x0F, 8); } -static inline uint64_t CVMX_CIU_MBOX_SETX(unsigned long offset) +static inline uint64_t CVMX_CIU_MBOX_SETX(unsigned int coreid) { - switch (cvmx_get_octeon_family()) { - case OCTEON_CN30XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN52XX & OCTEON_FAMILY_MASK: - case OCTEON_CNF71XX & OCTEON_FAMILY_MASK: - case OCTEON_CN61XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN31XX & OCTEON_FAMILY_MASK: - case OCTEON_CN50XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN38XX & OCTEON_FAMILY_MASK: - case OCTEON_CN58XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN56XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN66XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN63XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; - case OCTEON_CN68XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070100100400ull) + (offset) * 8; - } - return CVMX_ADD_IO_SEG(0x0001070000000600ull) + (offset) * 8; + if (cvmx_get_octeon_family() == (OCTEON_CN68XX & OCTEON_FAMILY_MASK)) + return CVMX_CIU_ADDR(0x100100400, coreid, 0x0F, 8); + else + return CVMX_CIU_ADDR(0x000000600, coreid, 0x0F, 8); } -#define CVMX_CIU_NMI (CVMX_ADD_IO_SEG(0x0001070000000718ull)) -#define CVMX_CIU_PCI_INTA (CVMX_ADD_IO_SEG(0x0001070000000750ull)) -#define CVMX_CIU_PP_BIST_STAT (CVMX_ADD_IO_SEG(0x00010700000007E0ull)) -#define CVMX_CIU_PP_DBG (CVMX_ADD_IO_SEG(0x0001070000000708ull)) -static inline uint64_t CVMX_CIU_PP_POKEX(unsigned long offset) +static inline uint64_t CVMX_CIU_PP_POKEX(unsigned int coreid) { switch (cvmx_get_octeon_family()) { - case OCTEON_CN30XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN52XX & OCTEON_FAMILY_MASK: - case OCTEON_CNF71XX & OCTEON_FAMILY_MASK: - case OCTEON_CN61XX & OCTEON_FAMILY_MASK: - case OCTEON_CN70XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN31XX & OCTEON_FAMILY_MASK: - case OCTEON_CN50XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN38XX & OCTEON_FAMILY_MASK: - case OCTEON_CN58XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN56XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN66XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; - case OCTEON_CN63XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; case OCTEON_CN68XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070100100200ull) + (offset) * 8; + return CVMX_CIU_ADDR(0x100100200, coreid, 0x0F, 8); case OCTEON_CNF75XX & OCTEON_FAMILY_MASK: case OCTEON_CN73XX & OCTEON_FAMILY_MASK: case OCTEON_CN78XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001010000030000ull) + (offset) * 8; + return CVMX_CIU_ADDR(0x000030000, coreid, 0x0F, 8) - + 0x60000000000ull; + default: + return CVMX_CIU_ADDR(0x000000580, coreid, 0x0F, 8); } - return CVMX_ADD_IO_SEG(0x0001070000000580ull) + (offset) * 8; } -#define CVMX_CIU_PP_RST (CVMX_ADD_IO_SEG(0x0001070000000700ull)) -#define CVMX_CIU_QLM0 (CVMX_ADD_IO_SEG(0x0001070000000780ull)) -#define CVMX_CIU_QLM1 (CVMX_ADD_IO_SEG(0x0001070000000788ull)) -#define CVMX_CIU_QLM2 (CVMX_ADD_IO_SEG(0x0001070000000790ull)) -#define CVMX_CIU_QLM3 (CVMX_ADD_IO_SEG(0x0001070000000798ull)) -#define CVMX_CIU_QLM4 (CVMX_ADD_IO_SEG(0x00010700000007A0ull)) -#define CVMX_CIU_QLM_DCOK (CVMX_ADD_IO_SEG(0x0001070000000760ull)) -#define CVMX_CIU_QLM_JTGC (CVMX_ADD_IO_SEG(0x0001070000000768ull)) -#define CVMX_CIU_QLM_JTGD (CVMX_ADD_IO_SEG(0x0001070000000770ull)) -#define CVMX_CIU_SOFT_BIST (CVMX_ADD_IO_SEG(0x0001070000000738ull)) -#define CVMX_CIU_SOFT_PRST (CVMX_ADD_IO_SEG(0x0001070000000748ull)) -#define CVMX_CIU_SOFT_PRST1 (CVMX_ADD_IO_SEG(0x0001070000000758ull)) -#define CVMX_CIU_SOFT_PRST2 (CVMX_ADD_IO_SEG(0x00010700000007D8ull)) -#define CVMX_CIU_SOFT_PRST3 (CVMX_ADD_IO_SEG(0x00010700000007E0ull)) -#define CVMX_CIU_SOFT_RST (CVMX_ADD_IO_SEG(0x0001070000000740ull)) -#define CVMX_CIU_SUM1_IOX_INT(offset) (CVMX_ADD_IO_SEG(0x0001070000008600ull) + ((offset) & 1) * 8) -#define CVMX_CIU_SUM1_PPX_IP2(offset) (CVMX_ADD_IO_SEG(0x0001070000008000ull) + ((offset) & 15) * 8) -#define CVMX_CIU_SUM1_PPX_IP3(offset) (CVMX_ADD_IO_SEG(0x0001070000008200ull) + ((offset) & 15) * 8) -#define CVMX_CIU_SUM1_PPX_IP4(offset) (CVMX_ADD_IO_SEG(0x0001070000008400ull) + ((offset) & 15) * 8) -#define CVMX_CIU_SUM2_IOX_INT(offset) (CVMX_ADD_IO_SEG(0x0001070000008E00ull) + ((offset) & 1) * 8) -#define CVMX_CIU_SUM2_PPX_IP2(offset) (CVMX_ADD_IO_SEG(0x0001070000008800ull) + ((offset) & 15) * 8) -#define CVMX_CIU_SUM2_PPX_IP3(offset) (CVMX_ADD_IO_SEG(0x0001070000008A00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_SUM2_PPX_IP4(offset) (CVMX_ADD_IO_SEG(0x0001070000008C00ull) + ((offset) & 15) * 8) -#define CVMX_CIU_TIMX(offset) (CVMX_ADD_IO_SEG(0x0001070000000480ull) + ((offset) & 15) * 8) -#define CVMX_CIU_TIM_MULTI_CAST (CVMX_ADD_IO_SEG(0x000107000000C200ull)) -static inline uint64_t CVMX_CIU_WDOGX(unsigned long offset) +static inline uint64_t CVMX_CIU_WDOGX(unsigned int coreid) { switch (cvmx_get_octeon_family()) { - case OCTEON_CN30XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN52XX & OCTEON_FAMILY_MASK: - case OCTEON_CNF71XX & OCTEON_FAMILY_MASK: - case OCTEON_CN61XX & OCTEON_FAMILY_MASK: - case OCTEON_CN70XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN31XX & OCTEON_FAMILY_MASK: - case OCTEON_CN50XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN38XX & OCTEON_FAMILY_MASK: - case OCTEON_CN58XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN56XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN66XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; - case OCTEON_CN63XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; case OCTEON_CN68XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001070100100000ull) + (offset) * 8; + return CVMX_CIU_ADDR(0x100100000, coreid, 0x0F, 8); case OCTEON_CNF75XX & OCTEON_FAMILY_MASK: case OCTEON_CN73XX & OCTEON_FAMILY_MASK: case OCTEON_CN78XX & OCTEON_FAMILY_MASK: - return CVMX_ADD_IO_SEG(0x0001010000020000ull) + (offset) * 8; + return CVMX_CIU_ADDR(0x000020000, coreid, 0x0F, 8) - + 0x60000000000ull; + default: + return CVMX_CIU_ADDR(0x000000500, coreid, 0x0F, 8); } - return CVMX_ADD_IO_SEG(0x0001070000000500ull) + (offset) * 8; } -union cvmx_ciu_bist { - uint64_t u64; - struct cvmx_ciu_bist_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_7_63:57; - uint64_t bist:7; -#else - uint64_t bist:7; - uint64_t reserved_7_63:57; -#endif - } s; - struct cvmx_ciu_bist_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t bist:4; -#else - uint64_t bist:4; - uint64_t reserved_4_63:60; -#endif - } cn30xx; - struct cvmx_ciu_bist_cn30xx cn31xx; - struct cvmx_ciu_bist_cn30xx cn38xx; - struct cvmx_ciu_bist_cn30xx cn38xxp2; - struct cvmx_ciu_bist_cn50xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t bist:2; -#else - uint64_t bist:2; - uint64_t reserved_2_63:62; -#endif - } cn50xx; - struct cvmx_ciu_bist_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_3_63:61; - uint64_t bist:3; -#else - uint64_t bist:3; - uint64_t reserved_3_63:61; -#endif - } cn52xx; - struct cvmx_ciu_bist_cn52xx cn52xxp1; - struct cvmx_ciu_bist_cn30xx cn56xx; - struct cvmx_ciu_bist_cn30xx cn56xxp1; - struct cvmx_ciu_bist_cn30xx cn58xx; - struct cvmx_ciu_bist_cn30xx cn58xxp1; - struct cvmx_ciu_bist_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t bist:6; -#else - uint64_t bist:6; - uint64_t reserved_6_63:58; -#endif - } cn61xx; - struct cvmx_ciu_bist_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_5_63:59; - uint64_t bist:5; -#else - uint64_t bist:5; - uint64_t reserved_5_63:59; -#endif - } cn63xx; - struct cvmx_ciu_bist_cn63xx cn63xxp1; - struct cvmx_ciu_bist_cn61xx cn66xx; - struct cvmx_ciu_bist_s cn68xx; - struct cvmx_ciu_bist_s cn68xxp1; - struct cvmx_ciu_bist_cn61xx cnf71xx; -}; - -union cvmx_ciu_block_int { - uint64_t u64; - struct cvmx_ciu_block_int_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_62_63:2; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_43_59:17; - uint64_t ptp:1; - uint64_t dpi:1; - uint64_t dfm:1; - uint64_t reserved_34_39:6; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t reserved_31_31:1; - uint64_t iob:1; - uint64_t reserved_29_29:1; - uint64_t agl:1; - uint64_t reserved_27_27:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t reserved_24_24:1; - uint64_t asxpcs1:1; - uint64_t asxpcs0:1; - uint64_t reserved_21_21:1; - uint64_t pip:1; - uint64_t reserved_18_19:2; - uint64_t lmc0:1; - uint64_t l2c:1; - uint64_t reserved_15_15:1; - uint64_t rad:1; - uint64_t usb:1; - uint64_t pow:1; - uint64_t tim:1; - uint64_t pko:1; - uint64_t ipd:1; - uint64_t reserved_8_8:1; - uint64_t zip:1; - uint64_t dfa:1; - uint64_t fpa:1; - uint64_t key:1; - uint64_t sli:1; - uint64_t gmx1:1; - uint64_t gmx0:1; - uint64_t mio:1; -#else - uint64_t mio:1; - uint64_t gmx0:1; - uint64_t gmx1:1; - uint64_t sli:1; - uint64_t key:1; - uint64_t fpa:1; - uint64_t dfa:1; - uint64_t zip:1; - uint64_t reserved_8_8:1; - uint64_t ipd:1; - uint64_t pko:1; - uint64_t tim:1; - uint64_t pow:1; - uint64_t usb:1; - uint64_t rad:1; - uint64_t reserved_15_15:1; - uint64_t l2c:1; - uint64_t lmc0:1; - uint64_t reserved_18_19:2; - uint64_t pip:1; - uint64_t reserved_21_21:1; - uint64_t asxpcs0:1; - uint64_t asxpcs1:1; - uint64_t reserved_24_24:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_27_27:1; - uint64_t agl:1; - uint64_t reserved_29_29:1; - uint64_t iob:1; - uint64_t reserved_31_31:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t reserved_34_39:6; - uint64_t dfm:1; - uint64_t dpi:1; - uint64_t ptp:1; - uint64_t reserved_43_59:17; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_63:2; -#endif - } s; - struct cvmx_ciu_block_int_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_43_63:21; - uint64_t ptp:1; - uint64_t dpi:1; - uint64_t reserved_31_40:10; - uint64_t iob:1; - uint64_t reserved_29_29:1; - uint64_t agl:1; - uint64_t reserved_27_27:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t reserved_24_24:1; - uint64_t asxpcs1:1; - uint64_t asxpcs0:1; - uint64_t reserved_21_21:1; - uint64_t pip:1; - uint64_t reserved_18_19:2; - uint64_t lmc0:1; - uint64_t l2c:1; - uint64_t reserved_15_15:1; - uint64_t rad:1; - uint64_t usb:1; - uint64_t pow:1; - uint64_t tim:1; - uint64_t pko:1; - uint64_t ipd:1; - uint64_t reserved_8_8:1; - uint64_t zip:1; - uint64_t dfa:1; - uint64_t fpa:1; - uint64_t key:1; - uint64_t sli:1; - uint64_t gmx1:1; - uint64_t gmx0:1; - uint64_t mio:1; -#else - uint64_t mio:1; - uint64_t gmx0:1; - uint64_t gmx1:1; - uint64_t sli:1; - uint64_t key:1; - uint64_t fpa:1; - uint64_t dfa:1; - uint64_t zip:1; - uint64_t reserved_8_8:1; - uint64_t ipd:1; - uint64_t pko:1; - uint64_t tim:1; - uint64_t pow:1; - uint64_t usb:1; - uint64_t rad:1; - uint64_t reserved_15_15:1; - uint64_t l2c:1; - uint64_t lmc0:1; - uint64_t reserved_18_19:2; - uint64_t pip:1; - uint64_t reserved_21_21:1; - uint64_t asxpcs0:1; - uint64_t asxpcs1:1; - uint64_t reserved_24_24:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_27_27:1; - uint64_t agl:1; - uint64_t reserved_29_29:1; - uint64_t iob:1; - uint64_t reserved_31_40:10; - uint64_t dpi:1; - uint64_t ptp:1; - uint64_t reserved_43_63:21; -#endif - } cn61xx; - struct cvmx_ciu_block_int_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_43_63:21; - uint64_t ptp:1; - uint64_t dpi:1; - uint64_t dfm:1; - uint64_t reserved_34_39:6; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t reserved_31_31:1; - uint64_t iob:1; - uint64_t reserved_29_29:1; - uint64_t agl:1; - uint64_t reserved_27_27:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t reserved_23_24:2; - uint64_t asxpcs0:1; - uint64_t reserved_21_21:1; - uint64_t pip:1; - uint64_t reserved_18_19:2; - uint64_t lmc0:1; - uint64_t l2c:1; - uint64_t reserved_15_15:1; - uint64_t rad:1; - uint64_t usb:1; - uint64_t pow:1; - uint64_t tim:1; - uint64_t pko:1; - uint64_t ipd:1; - uint64_t reserved_8_8:1; - uint64_t zip:1; - uint64_t dfa:1; - uint64_t fpa:1; - uint64_t key:1; - uint64_t sli:1; - uint64_t reserved_2_2:1; - uint64_t gmx0:1; - uint64_t mio:1; -#else - uint64_t mio:1; - uint64_t gmx0:1; - uint64_t reserved_2_2:1; - uint64_t sli:1; - uint64_t key:1; - uint64_t fpa:1; - uint64_t dfa:1; - uint64_t zip:1; - uint64_t reserved_8_8:1; - uint64_t ipd:1; - uint64_t pko:1; - uint64_t tim:1; - uint64_t pow:1; - uint64_t usb:1; - uint64_t rad:1; - uint64_t reserved_15_15:1; - uint64_t l2c:1; - uint64_t lmc0:1; - uint64_t reserved_18_19:2; - uint64_t pip:1; - uint64_t reserved_21_21:1; - uint64_t asxpcs0:1; - uint64_t reserved_23_24:2; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_27_27:1; - uint64_t agl:1; - uint64_t reserved_29_29:1; - uint64_t iob:1; - uint64_t reserved_31_31:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t reserved_34_39:6; - uint64_t dfm:1; - uint64_t dpi:1; - uint64_t ptp:1; - uint64_t reserved_43_63:21; -#endif - } cn63xx; - struct cvmx_ciu_block_int_cn63xx cn63xxp1; - struct cvmx_ciu_block_int_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_62_63:2; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_43_59:17; - uint64_t ptp:1; - uint64_t dpi:1; - uint64_t dfm:1; - uint64_t reserved_33_39:7; - uint64_t srio0:1; - uint64_t reserved_31_31:1; - uint64_t iob:1; - uint64_t reserved_29_29:1; - uint64_t agl:1; - uint64_t reserved_27_27:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t reserved_24_24:1; - uint64_t asxpcs1:1; - uint64_t asxpcs0:1; - uint64_t reserved_21_21:1; - uint64_t pip:1; - uint64_t reserved_18_19:2; - uint64_t lmc0:1; - uint64_t l2c:1; - uint64_t reserved_15_15:1; - uint64_t rad:1; - uint64_t usb:1; - uint64_t pow:1; - uint64_t tim:1; - uint64_t pko:1; - uint64_t ipd:1; - uint64_t reserved_8_8:1; - uint64_t zip:1; - uint64_t dfa:1; - uint64_t fpa:1; - uint64_t key:1; - uint64_t sli:1; - uint64_t gmx1:1; - uint64_t gmx0:1; - uint64_t mio:1; -#else - uint64_t mio:1; - uint64_t gmx0:1; - uint64_t gmx1:1; - uint64_t sli:1; - uint64_t key:1; - uint64_t fpa:1; - uint64_t dfa:1; - uint64_t zip:1; - uint64_t reserved_8_8:1; - uint64_t ipd:1; - uint64_t pko:1; - uint64_t tim:1; - uint64_t pow:1; - uint64_t usb:1; - uint64_t rad:1; - uint64_t reserved_15_15:1; - uint64_t l2c:1; - uint64_t lmc0:1; - uint64_t reserved_18_19:2; - uint64_t pip:1; - uint64_t reserved_21_21:1; - uint64_t asxpcs0:1; - uint64_t asxpcs1:1; - uint64_t reserved_24_24:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_27_27:1; - uint64_t agl:1; - uint64_t reserved_29_29:1; - uint64_t iob:1; - uint64_t reserved_31_31:1; - uint64_t srio0:1; - uint64_t reserved_33_39:7; - uint64_t dfm:1; - uint64_t dpi:1; - uint64_t ptp:1; - uint64_t reserved_43_59:17; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_63:2; -#endif - } cn66xx; - struct cvmx_ciu_block_int_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_43_63:21; - uint64_t ptp:1; - uint64_t dpi:1; - uint64_t reserved_31_40:10; - uint64_t iob:1; - uint64_t reserved_27_29:3; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t reserved_23_24:2; - uint64_t asxpcs0:1; - uint64_t reserved_21_21:1; - uint64_t pip:1; - uint64_t reserved_18_19:2; - uint64_t lmc0:1; - uint64_t l2c:1; - uint64_t reserved_15_15:1; - uint64_t rad:1; - uint64_t usb:1; - uint64_t pow:1; - uint64_t tim:1; - uint64_t pko:1; - uint64_t ipd:1; - uint64_t reserved_6_8:3; - uint64_t fpa:1; - uint64_t key:1; - uint64_t sli:1; - uint64_t reserved_2_2:1; - uint64_t gmx0:1; - uint64_t mio:1; -#else - uint64_t mio:1; - uint64_t gmx0:1; - uint64_t reserved_2_2:1; - uint64_t sli:1; - uint64_t key:1; - uint64_t fpa:1; - uint64_t reserved_6_8:3; - uint64_t ipd:1; - uint64_t pko:1; - uint64_t tim:1; - uint64_t pow:1; - uint64_t usb:1; - uint64_t rad:1; - uint64_t reserved_15_15:1; - uint64_t l2c:1; - uint64_t lmc0:1; - uint64_t reserved_18_19:2; - uint64_t pip:1; - uint64_t reserved_21_21:1; - uint64_t asxpcs0:1; - uint64_t reserved_23_24:2; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_27_29:3; - uint64_t iob:1; - uint64_t reserved_31_40:10; - uint64_t dpi:1; - uint64_t ptp:1; - uint64_t reserved_43_63:21; -#endif - } cnf71xx; -}; - -union cvmx_ciu_dint { - uint64_t u64; - struct cvmx_ciu_dint_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t dint:32; -#else - uint64_t dint:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_dint_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t dint:1; -#else - uint64_t dint:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_dint_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t dint:2; -#else - uint64_t dint:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_dint_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t dint:16; -#else - uint64_t dint:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_dint_cn38xx cn38xxp2; - struct cvmx_ciu_dint_cn31xx cn50xx; - struct cvmx_ciu_dint_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t dint:4; -#else - uint64_t dint:4; - uint64_t reserved_4_63:60; -#endif - } cn52xx; - struct cvmx_ciu_dint_cn52xx cn52xxp1; - struct cvmx_ciu_dint_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t dint:12; -#else - uint64_t dint:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_dint_cn56xx cn56xxp1; - struct cvmx_ciu_dint_cn38xx cn58xx; - struct cvmx_ciu_dint_cn38xx cn58xxp1; - struct cvmx_ciu_dint_cn52xx cn61xx; - struct cvmx_ciu_dint_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t dint:6; -#else - uint64_t dint:6; - uint64_t reserved_6_63:58; -#endif - } cn63xx; - struct cvmx_ciu_dint_cn63xx cn63xxp1; - struct cvmx_ciu_dint_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t dint:10; -#else - uint64_t dint:10; - uint64_t reserved_10_63:54; -#endif - } cn66xx; - struct cvmx_ciu_dint_s cn68xx; - struct cvmx_ciu_dint_s cn68xxp1; - struct cvmx_ciu_dint_cn52xx cnf71xx; -}; - -union cvmx_ciu_en2_iox_int { - uint64_t u64; - struct cvmx_ciu_en2_iox_int_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_iox_int_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_iox_int_cn61xx cn66xx; - struct cvmx_ciu_en2_iox_int_s cnf71xx; -}; - -union cvmx_ciu_en2_iox_int_w1c { - uint64_t u64; - struct cvmx_ciu_en2_iox_int_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_iox_int_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_iox_int_w1c_cn61xx cn66xx; - struct cvmx_ciu_en2_iox_int_w1c_s cnf71xx; -}; - -union cvmx_ciu_en2_iox_int_w1s { - uint64_t u64; - struct cvmx_ciu_en2_iox_int_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_iox_int_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_iox_int_w1s_cn61xx cn66xx; - struct cvmx_ciu_en2_iox_int_w1s_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip2 { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip2_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip2_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip2_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip2_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip2_w1c { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip2_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip2_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip2_w1c_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip2_w1c_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip2_w1s { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip2_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip2_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip2_w1s_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip2_w1s_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip3 { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip3_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip3_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip3_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip3_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip3_w1c { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip3_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip3_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip3_w1c_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip3_w1c_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip3_w1s { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip3_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip3_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip3_w1s_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip3_w1s_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip4 { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip4_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip4_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip4_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip4_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip4_w1c { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip4_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip4_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip4_w1c_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip4_w1c_s cnf71xx; -}; - -union cvmx_ciu_en2_ppx_ip4_w1s { - uint64_t u64; - struct cvmx_ciu_en2_ppx_ip4_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_en2_ppx_ip4_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_en2_ppx_ip4_w1s_cn61xx cn66xx; - struct cvmx_ciu_en2_ppx_ip4_w1s_s cnf71xx; -}; - -union cvmx_ciu_fuse { - uint64_t u64; - struct cvmx_ciu_fuse_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t fuse:32; -#else - uint64_t fuse:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_fuse_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t fuse:1; -#else - uint64_t fuse:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_fuse_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t fuse:2; -#else - uint64_t fuse:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_fuse_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t fuse:16; -#else - uint64_t fuse:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_fuse_cn38xx cn38xxp2; - struct cvmx_ciu_fuse_cn31xx cn50xx; - struct cvmx_ciu_fuse_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t fuse:4; -#else - uint64_t fuse:4; - uint64_t reserved_4_63:60; -#endif - } cn52xx; - struct cvmx_ciu_fuse_cn52xx cn52xxp1; - struct cvmx_ciu_fuse_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t fuse:12; -#else - uint64_t fuse:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_fuse_cn56xx cn56xxp1; - struct cvmx_ciu_fuse_cn38xx cn58xx; - struct cvmx_ciu_fuse_cn38xx cn58xxp1; - struct cvmx_ciu_fuse_cn52xx cn61xx; - struct cvmx_ciu_fuse_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t fuse:6; -#else - uint64_t fuse:6; - uint64_t reserved_6_63:58; -#endif - } cn63xx; - struct cvmx_ciu_fuse_cn63xx cn63xxp1; - struct cvmx_ciu_fuse_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t fuse:10; -#else - uint64_t fuse:10; - uint64_t reserved_10_63:54; -#endif - } cn66xx; - struct cvmx_ciu_fuse_s cn68xx; - struct cvmx_ciu_fuse_s cn68xxp1; - struct cvmx_ciu_fuse_cn52xx cnf71xx; -}; - -union cvmx_ciu_gstop { - uint64_t u64; - struct cvmx_ciu_gstop_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t gstop:1; -#else - uint64_t gstop:1; - uint64_t reserved_1_63:63; -#endif - } s; - struct cvmx_ciu_gstop_s cn30xx; - struct cvmx_ciu_gstop_s cn31xx; - struct cvmx_ciu_gstop_s cn38xx; - struct cvmx_ciu_gstop_s cn38xxp2; - struct cvmx_ciu_gstop_s cn50xx; - struct cvmx_ciu_gstop_s cn52xx; - struct cvmx_ciu_gstop_s cn52xxp1; - struct cvmx_ciu_gstop_s cn56xx; - struct cvmx_ciu_gstop_s cn56xxp1; - struct cvmx_ciu_gstop_s cn58xx; - struct cvmx_ciu_gstop_s cn58xxp1; - struct cvmx_ciu_gstop_s cn61xx; - struct cvmx_ciu_gstop_s cn63xx; - struct cvmx_ciu_gstop_s cn63xxp1; - struct cvmx_ciu_gstop_s cn66xx; - struct cvmx_ciu_gstop_s cn68xx; - struct cvmx_ciu_gstop_s cn68xxp1; - struct cvmx_ciu_gstop_s cnf71xx; -}; - -union cvmx_ciu_intx_en0 { - uint64_t u64; - struct cvmx_ciu_intx_en0_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en0_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t reserved_47_47:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t reserved_47_47:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn30xx; - struct cvmx_ciu_intx_en0_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn31xx; - struct cvmx_ciu_intx_en0_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn38xx; - struct cvmx_ciu_intx_en0_cn38xx cn38xxp2; - struct cvmx_ciu_intx_en0_cn30xx cn50xx; - struct cvmx_ciu_intx_en0_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en0_cn52xx cn52xxp1; - struct cvmx_ciu_intx_en0_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en0_cn56xx cn56xxp1; - struct cvmx_ciu_intx_en0_cn38xx cn58xx; - struct cvmx_ciu_intx_en0_cn38xx cn58xxp1; - struct cvmx_ciu_intx_en0_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en0_cn52xx cn63xx; - struct cvmx_ciu_intx_en0_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en0_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en0_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en0_w1c { - uint64_t u64; - struct cvmx_ciu_intx_en0_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en0_w1c_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en0_w1c_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en0_w1c_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_en0_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en0_w1c_cn52xx cn63xx; - struct cvmx_ciu_intx_en0_w1c_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en0_w1c_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en0_w1c_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en0_w1s { - uint64_t u64; - struct cvmx_ciu_intx_en0_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en0_w1s_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en0_w1s_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en0_w1s_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_en0_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en0_w1s_cn52xx cn63xx; - struct cvmx_ciu_intx_en0_w1s_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en0_w1s_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en0_w1s_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en1 { - uint64_t u64; - struct cvmx_ciu_intx_en1_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en1_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t wdog:1; -#else - uint64_t wdog:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_intx_en1_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t wdog:2; -#else - uint64_t wdog:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_intx_en1_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_intx_en1_cn38xx cn38xxp2; - struct cvmx_ciu_intx_en1_cn31xx cn50xx; - struct cvmx_ciu_intx_en1_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en1_cn52xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t reserved_19_63:45; -#endif - } cn52xxp1; - struct cvmx_ciu_intx_en1_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en1_cn56xx cn56xxp1; - struct cvmx_ciu_intx_en1_cn38xx cn58xx; - struct cvmx_ciu_intx_en1_cn38xx cn58xxp1; - struct cvmx_ciu_intx_en1_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en1_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en1_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en1_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en1_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en1_w1c { - uint64_t u64; - struct cvmx_ciu_intx_en1_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en1_w1c_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en1_w1c_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en1_w1c_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn58xx; - struct cvmx_ciu_intx_en1_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en1_w1c_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en1_w1c_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en1_w1c_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en1_w1c_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en1_w1s { - uint64_t u64; - struct cvmx_ciu_intx_en1_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en1_w1s_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en1_w1s_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en1_w1s_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn58xx; - struct cvmx_ciu_intx_en1_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en1_w1s_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en1_w1s_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en1_w1s_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en1_w1s_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_0 { - uint64_t u64; - struct cvmx_ciu_intx_en4_0_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en4_0_cn50xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t reserved_47_47:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t reserved_47_47:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn50xx; - struct cvmx_ciu_intx_en4_0_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_0_cn52xx cn52xxp1; - struct cvmx_ciu_intx_en4_0_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_0_cn56xx cn56xxp1; - struct cvmx_ciu_intx_en4_0_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_0_cn58xx cn58xxp1; - struct cvmx_ciu_intx_en4_0_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_0_cn52xx cn63xx; - struct cvmx_ciu_intx_en4_0_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en4_0_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_0_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_0_w1c { - uint64_t u64; - struct cvmx_ciu_intx_en4_0_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en4_0_w1c_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_0_w1c_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_0_w1c_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_0_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_0_w1c_cn52xx cn63xx; - struct cvmx_ciu_intx_en4_0_w1c_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en4_0_w1c_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_0_w1c_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_0_w1s { - uint64_t u64; - struct cvmx_ciu_intx_en4_0_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_en4_0_w1s_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_0_w1s_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_0_w1s_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_0_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_0_w1s_cn52xx cn63xx; - struct cvmx_ciu_intx_en4_0_w1s_cn52xx cn63xxp1; - struct cvmx_ciu_intx_en4_0_w1s_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_0_w1s_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t reserved_44_44:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t reserved_44_44:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_1 { - uint64_t u64; - struct cvmx_ciu_intx_en4_1_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en4_1_cn50xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t wdog:2; -#else - uint64_t wdog:2; - uint64_t reserved_2_63:62; -#endif - } cn50xx; - struct cvmx_ciu_intx_en4_1_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_1_cn52xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t reserved_19_63:45; -#endif - } cn52xxp1; - struct cvmx_ciu_intx_en4_1_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_1_cn56xx cn56xxp1; - struct cvmx_ciu_intx_en4_1_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_1_cn58xx cn58xxp1; - struct cvmx_ciu_intx_en4_1_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_1_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en4_1_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en4_1_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_1_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_1_w1c { - uint64_t u64; - struct cvmx_ciu_intx_en4_1_w1c_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en4_1_w1c_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_1_w1c_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_1_w1c_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_1_w1c_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_1_w1c_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en4_1_w1c_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en4_1_w1c_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_1_w1c_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_en4_1_w1s { - uint64_t u64; - struct cvmx_ciu_intx_en4_1_w1s_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_intx_en4_1_w1s_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_intx_en4_1_w1s_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_intx_en4_1_w1s_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn58xx; - struct cvmx_ciu_intx_en4_1_w1s_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_en4_1_w1s_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_intx_en4_1_w1s_cn63xx cn63xxp1; - struct cvmx_ciu_intx_en4_1_w1s_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_en4_1_w1s_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_sum0 { - uint64_t u64; - struct cvmx_ciu_intx_sum0_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_sum0_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t reserved_47_47:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t reserved_47_47:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn30xx; - struct cvmx_ciu_intx_sum0_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn31xx; - struct cvmx_ciu_intx_sum0_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn38xx; - struct cvmx_ciu_intx_sum0_cn38xx cn38xxp2; - struct cvmx_ciu_intx_sum0_cn30xx cn50xx; - struct cvmx_ciu_intx_sum0_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_sum0_cn52xx cn52xxp1; - struct cvmx_ciu_intx_sum0_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_sum0_cn56xx cn56xxp1; - struct cvmx_ciu_intx_sum0_cn38xx cn58xx; - struct cvmx_ciu_intx_sum0_cn38xx cn58xxp1; - struct cvmx_ciu_intx_sum0_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_sum0_cn52xx cn63xx; - struct cvmx_ciu_intx_sum0_cn52xx cn63xxp1; - struct cvmx_ciu_intx_sum0_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_sum0_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_intx_sum4 { - uint64_t u64; - struct cvmx_ciu_intx_sum4_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_intx_sum4_cn50xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_59_63:5; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t reserved_47_47:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t reserved_47_47:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t reserved_59_63:5; -#endif - } cn50xx; - struct cvmx_ciu_intx_sum4_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn52xx; - struct cvmx_ciu_intx_sum4_cn52xx cn52xxp1; - struct cvmx_ciu_intx_sum4_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn56xx; - struct cvmx_ciu_intx_sum4_cn56xx cn56xxp1; - struct cvmx_ciu_intx_sum4_cn58xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_56_63:8; - uint64_t timer:4; - uint64_t key_zero:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t key_zero:1; - uint64_t timer:4; - uint64_t reserved_56_63:8; -#endif - } cn58xx; - struct cvmx_ciu_intx_sum4_cn58xx cn58xxp1; - struct cvmx_ciu_intx_sum4_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn61xx; - struct cvmx_ciu_intx_sum4_cn52xx cn63xx; - struct cvmx_ciu_intx_sum4_cn52xx cn63xxp1; - struct cvmx_ciu_intx_sum4_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_intx_sum4_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_int33_sum0 { - uint64_t u64; - struct cvmx_ciu_int33_sum0_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } s; - struct cvmx_ciu_int33_sum0_s cn61xx; - struct cvmx_ciu_int33_sum0_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t reserved_57_58:2; - uint64_t usb:1; - uint64_t timer:4; - uint64_t reserved_51_51:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t reserved_51_51:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_58:2; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn63xx; - struct cvmx_ciu_int33_sum0_cn63xx cn63xxp1; - struct cvmx_ciu_int33_sum0_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t mii:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t reserved_57_57:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t gmx_drp:2; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:2; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t reserved_57_57:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t mii:1; - uint64_t bootdma:1; -#endif - } cn66xx; - struct cvmx_ciu_int33_sum0_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t bootdma:1; - uint64_t reserved_62_62:1; - uint64_t ipdppthr:1; - uint64_t powiq:1; - uint64_t twsi2:1; - uint64_t mpi:1; - uint64_t pcm:1; - uint64_t usb:1; - uint64_t timer:4; - uint64_t sum2:1; - uint64_t ipd_drp:1; - uint64_t reserved_49_49:1; - uint64_t gmx_drp:1; - uint64_t trace:1; - uint64_t rml:1; - uint64_t twsi:1; - uint64_t wdog_sum:1; - uint64_t pci_msi:4; - uint64_t pci_int:4; - uint64_t uart:2; - uint64_t mbox:2; - uint64_t gpio:16; - uint64_t workq:16; -#else - uint64_t workq:16; - uint64_t gpio:16; - uint64_t mbox:2; - uint64_t uart:2; - uint64_t pci_int:4; - uint64_t pci_msi:4; - uint64_t wdog_sum:1; - uint64_t twsi:1; - uint64_t rml:1; - uint64_t trace:1; - uint64_t gmx_drp:1; - uint64_t reserved_49_49:1; - uint64_t ipd_drp:1; - uint64_t sum2:1; - uint64_t timer:4; - uint64_t usb:1; - uint64_t pcm:1; - uint64_t mpi:1; - uint64_t twsi2:1; - uint64_t powiq:1; - uint64_t ipdppthr:1; - uint64_t reserved_62_62:1; - uint64_t bootdma:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_int_dbg_sel { - uint64_t u64; - struct cvmx_ciu_int_dbg_sel_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t sel:3; - uint64_t reserved_10_15:6; - uint64_t irq:2; - uint64_t reserved_5_7:3; - uint64_t pp:5; -#else - uint64_t pp:5; - uint64_t reserved_5_7:3; - uint64_t irq:2; - uint64_t reserved_10_15:6; - uint64_t sel:3; - uint64_t reserved_19_63:45; -#endif - } s; - struct cvmx_ciu_int_dbg_sel_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t sel:3; - uint64_t reserved_10_15:6; - uint64_t irq:2; - uint64_t reserved_4_7:4; - uint64_t pp:4; -#else - uint64_t pp:4; - uint64_t reserved_4_7:4; - uint64_t irq:2; - uint64_t reserved_10_15:6; - uint64_t sel:3; - uint64_t reserved_19_63:45; -#endif - } cn61xx; - struct cvmx_ciu_int_dbg_sel_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t sel:3; - uint64_t reserved_10_15:6; - uint64_t irq:2; - uint64_t reserved_3_7:5; - uint64_t pp:3; -#else - uint64_t pp:3; - uint64_t reserved_3_7:5; - uint64_t irq:2; - uint64_t reserved_10_15:6; - uint64_t sel:3; - uint64_t reserved_19_63:45; -#endif - } cn63xx; - struct cvmx_ciu_int_dbg_sel_cn61xx cn66xx; - struct cvmx_ciu_int_dbg_sel_s cn68xx; - struct cvmx_ciu_int_dbg_sel_s cn68xxp1; - struct cvmx_ciu_int_dbg_sel_cn61xx cnf71xx; -}; - -union cvmx_ciu_int_sum1 { - uint64_t u64; - struct cvmx_ciu_int_sum1_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_int_sum1_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t wdog:1; -#else - uint64_t wdog:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_int_sum1_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t wdog:2; -#else - uint64_t wdog:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_int_sum1_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t wdog:16; -#else - uint64_t wdog:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_int_sum1_cn38xx cn38xxp2; - struct cvmx_ciu_int_sum1_cn31xx cn50xx; - struct cvmx_ciu_int_sum1_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_20_63:44; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t reserved_20_63:44; -#endif - } cn52xx; - struct cvmx_ciu_int_sum1_cn52xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_19_63:45; - uint64_t mii1:1; - uint64_t usb1:1; - uint64_t uart2:1; - uint64_t reserved_4_15:12; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_15:12; - uint64_t uart2:1; - uint64_t usb1:1; - uint64_t mii1:1; - uint64_t reserved_19_63:45; -#endif - } cn52xxp1; - struct cvmx_ciu_int_sum1_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t wdog:12; -#else - uint64_t wdog:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_int_sum1_cn56xx cn56xxp1; - struct cvmx_ciu_int_sum1_cn38xx cn58xx; - struct cvmx_ciu_int_sum1_cn38xx cn58xxp1; - struct cvmx_ciu_int_sum1_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_int_sum1_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_57_62:6; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t srio1:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_37_45:9; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_6_17:12; - uint64_t wdog:6; -#else - uint64_t wdog:6; - uint64_t reserved_6_17:12; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_45:9; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t srio1:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_62:6; - uint64_t rst:1; -#endif - } cn63xx; - struct cvmx_ciu_int_sum1_cn63xx cn63xxp1; - struct cvmx_ciu_int_sum1_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_int_sum1_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_37_46:10; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_46:10; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_mbox_clrx { - uint64_t u64; - struct cvmx_ciu_mbox_clrx_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t bits:32; -#else - uint64_t bits:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_mbox_clrx_s cn30xx; - struct cvmx_ciu_mbox_clrx_s cn31xx; - struct cvmx_ciu_mbox_clrx_s cn38xx; - struct cvmx_ciu_mbox_clrx_s cn38xxp2; - struct cvmx_ciu_mbox_clrx_s cn50xx; - struct cvmx_ciu_mbox_clrx_s cn52xx; - struct cvmx_ciu_mbox_clrx_s cn52xxp1; - struct cvmx_ciu_mbox_clrx_s cn56xx; - struct cvmx_ciu_mbox_clrx_s cn56xxp1; - struct cvmx_ciu_mbox_clrx_s cn58xx; - struct cvmx_ciu_mbox_clrx_s cn58xxp1; - struct cvmx_ciu_mbox_clrx_s cn61xx; - struct cvmx_ciu_mbox_clrx_s cn63xx; - struct cvmx_ciu_mbox_clrx_s cn63xxp1; - struct cvmx_ciu_mbox_clrx_s cn66xx; - struct cvmx_ciu_mbox_clrx_s cn68xx; - struct cvmx_ciu_mbox_clrx_s cn68xxp1; - struct cvmx_ciu_mbox_clrx_s cnf71xx; -}; - -union cvmx_ciu_mbox_setx { - uint64_t u64; - struct cvmx_ciu_mbox_setx_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t bits:32; -#else - uint64_t bits:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_mbox_setx_s cn30xx; - struct cvmx_ciu_mbox_setx_s cn31xx; - struct cvmx_ciu_mbox_setx_s cn38xx; - struct cvmx_ciu_mbox_setx_s cn38xxp2; - struct cvmx_ciu_mbox_setx_s cn50xx; - struct cvmx_ciu_mbox_setx_s cn52xx; - struct cvmx_ciu_mbox_setx_s cn52xxp1; - struct cvmx_ciu_mbox_setx_s cn56xx; - struct cvmx_ciu_mbox_setx_s cn56xxp1; - struct cvmx_ciu_mbox_setx_s cn58xx; - struct cvmx_ciu_mbox_setx_s cn58xxp1; - struct cvmx_ciu_mbox_setx_s cn61xx; - struct cvmx_ciu_mbox_setx_s cn63xx; - struct cvmx_ciu_mbox_setx_s cn63xxp1; - struct cvmx_ciu_mbox_setx_s cn66xx; - struct cvmx_ciu_mbox_setx_s cn68xx; - struct cvmx_ciu_mbox_setx_s cn68xxp1; - struct cvmx_ciu_mbox_setx_s cnf71xx; -}; - -union cvmx_ciu_nmi { - uint64_t u64; - struct cvmx_ciu_nmi_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t nmi:32; -#else - uint64_t nmi:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_nmi_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t nmi:1; -#else - uint64_t nmi:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_nmi_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t nmi:2; -#else - uint64_t nmi:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_nmi_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t nmi:16; -#else - uint64_t nmi:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_nmi_cn38xx cn38xxp2; - struct cvmx_ciu_nmi_cn31xx cn50xx; - struct cvmx_ciu_nmi_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t nmi:4; -#else - uint64_t nmi:4; - uint64_t reserved_4_63:60; -#endif - } cn52xx; - struct cvmx_ciu_nmi_cn52xx cn52xxp1; - struct cvmx_ciu_nmi_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t nmi:12; -#else - uint64_t nmi:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_nmi_cn56xx cn56xxp1; - struct cvmx_ciu_nmi_cn38xx cn58xx; - struct cvmx_ciu_nmi_cn38xx cn58xxp1; - struct cvmx_ciu_nmi_cn52xx cn61xx; - struct cvmx_ciu_nmi_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t nmi:6; -#else - uint64_t nmi:6; - uint64_t reserved_6_63:58; -#endif - } cn63xx; - struct cvmx_ciu_nmi_cn63xx cn63xxp1; - struct cvmx_ciu_nmi_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t nmi:10; -#else - uint64_t nmi:10; - uint64_t reserved_10_63:54; -#endif - } cn66xx; - struct cvmx_ciu_nmi_s cn68xx; - struct cvmx_ciu_nmi_s cn68xxp1; - struct cvmx_ciu_nmi_cn52xx cnf71xx; -}; - -union cvmx_ciu_pci_inta { - uint64_t u64; - struct cvmx_ciu_pci_inta_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t intr:2; -#else - uint64_t intr:2; - uint64_t reserved_2_63:62; -#endif - } s; - struct cvmx_ciu_pci_inta_s cn30xx; - struct cvmx_ciu_pci_inta_s cn31xx; - struct cvmx_ciu_pci_inta_s cn38xx; - struct cvmx_ciu_pci_inta_s cn38xxp2; - struct cvmx_ciu_pci_inta_s cn50xx; - struct cvmx_ciu_pci_inta_s cn52xx; - struct cvmx_ciu_pci_inta_s cn52xxp1; - struct cvmx_ciu_pci_inta_s cn56xx; - struct cvmx_ciu_pci_inta_s cn56xxp1; - struct cvmx_ciu_pci_inta_s cn58xx; - struct cvmx_ciu_pci_inta_s cn58xxp1; - struct cvmx_ciu_pci_inta_s cn61xx; - struct cvmx_ciu_pci_inta_s cn63xx; - struct cvmx_ciu_pci_inta_s cn63xxp1; - struct cvmx_ciu_pci_inta_s cn66xx; - struct cvmx_ciu_pci_inta_s cn68xx; - struct cvmx_ciu_pci_inta_s cn68xxp1; - struct cvmx_ciu_pci_inta_s cnf71xx; -}; - -union cvmx_ciu_pp_bist_stat { - uint64_t u64; - struct cvmx_ciu_pp_bist_stat_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t pp_bist:32; -#else - uint64_t pp_bist:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_pp_bist_stat_s cn68xx; - struct cvmx_ciu_pp_bist_stat_s cn68xxp1; -}; - -union cvmx_ciu_pp_dbg { - uint64_t u64; - struct cvmx_ciu_pp_dbg_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t ppdbg:32; -#else - uint64_t ppdbg:32; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_pp_dbg_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t ppdbg:1; -#else - uint64_t ppdbg:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_pp_dbg_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t ppdbg:2; -#else - uint64_t ppdbg:2; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_pp_dbg_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t ppdbg:16; -#else - uint64_t ppdbg:16; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_pp_dbg_cn38xx cn38xxp2; - struct cvmx_ciu_pp_dbg_cn31xx cn50xx; - struct cvmx_ciu_pp_dbg_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t ppdbg:4; -#else - uint64_t ppdbg:4; - uint64_t reserved_4_63:60; -#endif - } cn52xx; - struct cvmx_ciu_pp_dbg_cn52xx cn52xxp1; - struct cvmx_ciu_pp_dbg_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t ppdbg:12; -#else - uint64_t ppdbg:12; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_pp_dbg_cn56xx cn56xxp1; - struct cvmx_ciu_pp_dbg_cn38xx cn58xx; - struct cvmx_ciu_pp_dbg_cn38xx cn58xxp1; - struct cvmx_ciu_pp_dbg_cn52xx cn61xx; - struct cvmx_ciu_pp_dbg_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t ppdbg:6; -#else - uint64_t ppdbg:6; - uint64_t reserved_6_63:58; -#endif - } cn63xx; - struct cvmx_ciu_pp_dbg_cn63xx cn63xxp1; - struct cvmx_ciu_pp_dbg_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t ppdbg:10; -#else - uint64_t ppdbg:10; - uint64_t reserved_10_63:54; -#endif - } cn66xx; - struct cvmx_ciu_pp_dbg_s cn68xx; - struct cvmx_ciu_pp_dbg_s cn68xxp1; - struct cvmx_ciu_pp_dbg_cn52xx cnf71xx; -}; - -union cvmx_ciu_pp_pokex { - uint64_t u64; - struct cvmx_ciu_pp_pokex_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t poke:64; -#else - uint64_t poke:64; -#endif - } s; - struct cvmx_ciu_pp_pokex_s cn30xx; - struct cvmx_ciu_pp_pokex_s cn31xx; - struct cvmx_ciu_pp_pokex_s cn38xx; - struct cvmx_ciu_pp_pokex_s cn38xxp2; - struct cvmx_ciu_pp_pokex_s cn50xx; - struct cvmx_ciu_pp_pokex_s cn52xx; - struct cvmx_ciu_pp_pokex_s cn52xxp1; - struct cvmx_ciu_pp_pokex_s cn56xx; - struct cvmx_ciu_pp_pokex_s cn56xxp1; - struct cvmx_ciu_pp_pokex_s cn58xx; - struct cvmx_ciu_pp_pokex_s cn58xxp1; - struct cvmx_ciu_pp_pokex_s cn61xx; - struct cvmx_ciu_pp_pokex_s cn63xx; - struct cvmx_ciu_pp_pokex_s cn63xxp1; - struct cvmx_ciu_pp_pokex_s cn66xx; - struct cvmx_ciu_pp_pokex_s cn68xx; - struct cvmx_ciu_pp_pokex_s cn68xxp1; - struct cvmx_ciu_pp_pokex_s cnf71xx; -}; - -union cvmx_ciu_pp_rst { - uint64_t u64; - struct cvmx_ciu_pp_rst_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t rst:31; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:31; - uint64_t reserved_32_63:32; -#endif - } s; - struct cvmx_ciu_pp_rst_cn30xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t reserved_1_63:63; -#endif - } cn30xx; - struct cvmx_ciu_pp_rst_cn31xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t rst:1; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:1; - uint64_t reserved_2_63:62; -#endif - } cn31xx; - struct cvmx_ciu_pp_rst_cn38xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_16_63:48; - uint64_t rst:15; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:15; - uint64_t reserved_16_63:48; -#endif - } cn38xx; - struct cvmx_ciu_pp_rst_cn38xx cn38xxp2; - struct cvmx_ciu_pp_rst_cn31xx cn50xx; - struct cvmx_ciu_pp_rst_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t rst:3; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:3; - uint64_t reserved_4_63:60; -#endif - } cn52xx; - struct cvmx_ciu_pp_rst_cn52xx cn52xxp1; - struct cvmx_ciu_pp_rst_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_12_63:52; - uint64_t rst:11; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:11; - uint64_t reserved_12_63:52; -#endif - } cn56xx; - struct cvmx_ciu_pp_rst_cn56xx cn56xxp1; - struct cvmx_ciu_pp_rst_cn38xx cn58xx; - struct cvmx_ciu_pp_rst_cn38xx cn58xxp1; - struct cvmx_ciu_pp_rst_cn52xx cn61xx; - struct cvmx_ciu_pp_rst_cn63xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_6_63:58; - uint64_t rst:5; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:5; - uint64_t reserved_6_63:58; -#endif - } cn63xx; - struct cvmx_ciu_pp_rst_cn63xx cn63xxp1; - struct cvmx_ciu_pp_rst_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t rst:9; - uint64_t rst0:1; -#else - uint64_t rst0:1; - uint64_t rst:9; - uint64_t reserved_10_63:54; -#endif - } cn66xx; - struct cvmx_ciu_pp_rst_s cn68xx; - struct cvmx_ciu_pp_rst_s cn68xxp1; - struct cvmx_ciu_pp_rst_cn52xx cnf71xx; -}; - -union cvmx_ciu_qlm0 { - uint64_t u64; - struct cvmx_ciu_qlm0_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t g2bypass:1; - uint64_t reserved_53_62:10; - uint64_t g2deemph:5; - uint64_t reserved_45_47:3; - uint64_t g2margin:5; - uint64_t reserved_32_39:8; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_39:8; - uint64_t g2margin:5; - uint64_t reserved_45_47:3; - uint64_t g2deemph:5; - uint64_t reserved_53_62:10; - uint64_t g2bypass:1; -#endif - } s; - struct cvmx_ciu_qlm0_s cn61xx; - struct cvmx_ciu_qlm0_s cn63xx; - struct cvmx_ciu_qlm0_cn63xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t txbypass:1; - uint64_t reserved_20_30:11; - uint64_t txdeemph:4; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:4; - uint64_t reserved_20_30:11; - uint64_t txbypass:1; - uint64_t reserved_32_63:32; -#endif - } cn63xxp1; - struct cvmx_ciu_qlm0_s cn66xx; - struct cvmx_ciu_qlm0_cn68xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_63:32; -#endif - } cn68xx; - struct cvmx_ciu_qlm0_cn68xx cn68xxp1; - struct cvmx_ciu_qlm0_s cnf71xx; -}; - -union cvmx_ciu_qlm1 { - uint64_t u64; - struct cvmx_ciu_qlm1_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t g2bypass:1; - uint64_t reserved_53_62:10; - uint64_t g2deemph:5; - uint64_t reserved_45_47:3; - uint64_t g2margin:5; - uint64_t reserved_32_39:8; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_39:8; - uint64_t g2margin:5; - uint64_t reserved_45_47:3; - uint64_t g2deemph:5; - uint64_t reserved_53_62:10; - uint64_t g2bypass:1; -#endif - } s; - struct cvmx_ciu_qlm1_s cn61xx; - struct cvmx_ciu_qlm1_s cn63xx; - struct cvmx_ciu_qlm1_cn63xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t txbypass:1; - uint64_t reserved_20_30:11; - uint64_t txdeemph:4; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:4; - uint64_t reserved_20_30:11; - uint64_t txbypass:1; - uint64_t reserved_32_63:32; -#endif - } cn63xxp1; - struct cvmx_ciu_qlm1_s cn66xx; - struct cvmx_ciu_qlm1_s cn68xx; - struct cvmx_ciu_qlm1_s cn68xxp1; - struct cvmx_ciu_qlm1_s cnf71xx; -}; - -union cvmx_ciu_qlm2 { - uint64_t u64; - struct cvmx_ciu_qlm2_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t g2bypass:1; - uint64_t reserved_53_62:10; - uint64_t g2deemph:5; - uint64_t reserved_45_47:3; - uint64_t g2margin:5; - uint64_t reserved_32_39:8; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_39:8; - uint64_t g2margin:5; - uint64_t reserved_45_47:3; - uint64_t g2deemph:5; - uint64_t reserved_53_62:10; - uint64_t g2bypass:1; -#endif - } s; - struct cvmx_ciu_qlm2_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_63:32; -#endif - } cn61xx; - struct cvmx_ciu_qlm2_cn61xx cn63xx; - struct cvmx_ciu_qlm2_cn63xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_32_63:32; - uint64_t txbypass:1; - uint64_t reserved_20_30:11; - uint64_t txdeemph:4; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:4; - uint64_t reserved_20_30:11; - uint64_t txbypass:1; - uint64_t reserved_32_63:32; -#endif - } cn63xxp1; - struct cvmx_ciu_qlm2_cn61xx cn66xx; - struct cvmx_ciu_qlm2_s cn68xx; - struct cvmx_ciu_qlm2_s cn68xxp1; - struct cvmx_ciu_qlm2_cn61xx cnf71xx; -}; -union cvmx_ciu_qlm3 { +union cvmx_ciu_qlm { uint64_t u64; - struct cvmx_ciu_qlm3_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t g2bypass:1; - uint64_t reserved_53_62:10; - uint64_t g2deemph:5; - uint64_t reserved_45_47:3; - uint64_t g2margin:5; - uint64_t reserved_32_39:8; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_39:8; - uint64_t g2margin:5; - uint64_t reserved_45_47:3; - uint64_t g2deemph:5; - uint64_t reserved_53_62:10; - uint64_t g2bypass:1; -#endif + struct cvmx_ciu_qlm_s { + __BITFIELD_FIELD(uint64_t g2bypass:1, + __BITFIELD_FIELD(uint64_t reserved_53_62:10, + __BITFIELD_FIELD(uint64_t g2deemph:5, + __BITFIELD_FIELD(uint64_t reserved_45_47:3, + __BITFIELD_FIELD(uint64_t g2margin:5, + __BITFIELD_FIELD(uint64_t reserved_32_39:8, + __BITFIELD_FIELD(uint64_t txbypass:1, + __BITFIELD_FIELD(uint64_t reserved_21_30:10, + __BITFIELD_FIELD(uint64_t txdeemph:5, + __BITFIELD_FIELD(uint64_t reserved_13_15:3, + __BITFIELD_FIELD(uint64_t txmargin:5, + __BITFIELD_FIELD(uint64_t reserved_4_7:4, + __BITFIELD_FIELD(uint64_t lane_en:4, + ;))))))))))))) } s; - struct cvmx_ciu_qlm3_s cn68xx; - struct cvmx_ciu_qlm3_s cn68xxp1; -}; - -union cvmx_ciu_qlm4 { - uint64_t u64; - struct cvmx_ciu_qlm4_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t g2bypass:1; - uint64_t reserved_53_62:10; - uint64_t g2deemph:5; - uint64_t reserved_45_47:3; - uint64_t g2margin:5; - uint64_t reserved_32_39:8; - uint64_t txbypass:1; - uint64_t reserved_21_30:10; - uint64_t txdeemph:5; - uint64_t reserved_13_15:3; - uint64_t txmargin:5; - uint64_t reserved_4_7:4; - uint64_t lane_en:4; -#else - uint64_t lane_en:4; - uint64_t reserved_4_7:4; - uint64_t txmargin:5; - uint64_t reserved_13_15:3; - uint64_t txdeemph:5; - uint64_t reserved_21_30:10; - uint64_t txbypass:1; - uint64_t reserved_32_39:8; - uint64_t g2margin:5; - uint64_t reserved_45_47:3; - uint64_t g2deemph:5; - uint64_t reserved_53_62:10; - uint64_t g2bypass:1; -#endif - } s; - struct cvmx_ciu_qlm4_s cn68xx; - struct cvmx_ciu_qlm4_s cn68xxp1; -}; - -union cvmx_ciu_qlm_dcok { - uint64_t u64; - struct cvmx_ciu_qlm_dcok_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_4_63:60; - uint64_t qlm_dcok:4; -#else - uint64_t qlm_dcok:4; - uint64_t reserved_4_63:60; -#endif - } s; - struct cvmx_ciu_qlm_dcok_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_2_63:62; - uint64_t qlm_dcok:2; -#else - uint64_t qlm_dcok:2; - uint64_t reserved_2_63:62; -#endif - } cn52xx; - struct cvmx_ciu_qlm_dcok_cn52xx cn52xxp1; - struct cvmx_ciu_qlm_dcok_s cn56xx; - struct cvmx_ciu_qlm_dcok_s cn56xxp1; }; union cvmx_ciu_qlm_jtgc { uint64_t u64; struct cvmx_ciu_qlm_jtgc_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_17_63:47; - uint64_t bypass_ext:1; - uint64_t reserved_11_15:5; - uint64_t clk_div:3; - uint64_t reserved_7_7:1; - uint64_t mux_sel:3; - uint64_t bypass:4; -#else - uint64_t bypass:4; - uint64_t mux_sel:3; - uint64_t reserved_7_7:1; - uint64_t clk_div:3; - uint64_t reserved_11_15:5; - uint64_t bypass_ext:1; - uint64_t reserved_17_63:47; -#endif + __BITFIELD_FIELD(uint64_t reserved_17_63:47, + __BITFIELD_FIELD(uint64_t bypass_ext:1, + __BITFIELD_FIELD(uint64_t reserved_11_15:5, + __BITFIELD_FIELD(uint64_t clk_div:3, + __BITFIELD_FIELD(uint64_t reserved_7_7:1, + __BITFIELD_FIELD(uint64_t mux_sel:3, + __BITFIELD_FIELD(uint64_t bypass:4, + ;))))))) } s; - struct cvmx_ciu_qlm_jtgc_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_11_63:53; - uint64_t clk_div:3; - uint64_t reserved_5_7:3; - uint64_t mux_sel:1; - uint64_t reserved_2_3:2; - uint64_t bypass:2; -#else - uint64_t bypass:2; - uint64_t reserved_2_3:2; - uint64_t mux_sel:1; - uint64_t reserved_5_7:3; - uint64_t clk_div:3; - uint64_t reserved_11_63:53; -#endif - } cn52xx; - struct cvmx_ciu_qlm_jtgc_cn52xx cn52xxp1; - struct cvmx_ciu_qlm_jtgc_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_11_63:53; - uint64_t clk_div:3; - uint64_t reserved_6_7:2; - uint64_t mux_sel:2; - uint64_t bypass:4; -#else - uint64_t bypass:4; - uint64_t mux_sel:2; - uint64_t reserved_6_7:2; - uint64_t clk_div:3; - uint64_t reserved_11_63:53; -#endif - } cn56xx; - struct cvmx_ciu_qlm_jtgc_cn56xx cn56xxp1; - struct cvmx_ciu_qlm_jtgc_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_11_63:53; - uint64_t clk_div:3; - uint64_t reserved_6_7:2; - uint64_t mux_sel:2; - uint64_t reserved_3_3:1; - uint64_t bypass:3; -#else - uint64_t bypass:3; - uint64_t reserved_3_3:1; - uint64_t mux_sel:2; - uint64_t reserved_6_7:2; - uint64_t clk_div:3; - uint64_t reserved_11_63:53; -#endif - } cn61xx; - struct cvmx_ciu_qlm_jtgc_cn61xx cn63xx; - struct cvmx_ciu_qlm_jtgc_cn61xx cn63xxp1; - struct cvmx_ciu_qlm_jtgc_cn61xx cn66xx; - struct cvmx_ciu_qlm_jtgc_s cn68xx; - struct cvmx_ciu_qlm_jtgc_s cn68xxp1; - struct cvmx_ciu_qlm_jtgc_cn61xx cnf71xx; }; union cvmx_ciu_qlm_jtgd { uint64_t u64; struct cvmx_ciu_qlm_jtgd_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t capture:1; - uint64_t shift:1; - uint64_t update:1; - uint64_t reserved_45_60:16; - uint64_t select:5; - uint64_t reserved_37_39:3; - uint64_t shft_cnt:5; - uint64_t shft_reg:32; -#else - uint64_t shft_reg:32; - uint64_t shft_cnt:5; - uint64_t reserved_37_39:3; - uint64_t select:5; - uint64_t reserved_45_60:16; - uint64_t update:1; - uint64_t shift:1; - uint64_t capture:1; -#endif - } s; - struct cvmx_ciu_qlm_jtgd_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t capture:1; - uint64_t shift:1; - uint64_t update:1; - uint64_t reserved_42_60:19; - uint64_t select:2; - uint64_t reserved_37_39:3; - uint64_t shft_cnt:5; - uint64_t shft_reg:32; -#else - uint64_t shft_reg:32; - uint64_t shft_cnt:5; - uint64_t reserved_37_39:3; - uint64_t select:2; - uint64_t reserved_42_60:19; - uint64_t update:1; - uint64_t shift:1; - uint64_t capture:1; -#endif - } cn52xx; - struct cvmx_ciu_qlm_jtgd_cn52xx cn52xxp1; - struct cvmx_ciu_qlm_jtgd_cn56xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t capture:1; - uint64_t shift:1; - uint64_t update:1; - uint64_t reserved_44_60:17; - uint64_t select:4; - uint64_t reserved_37_39:3; - uint64_t shft_cnt:5; - uint64_t shft_reg:32; -#else - uint64_t shft_reg:32; - uint64_t shft_cnt:5; - uint64_t reserved_37_39:3; - uint64_t select:4; - uint64_t reserved_44_60:17; - uint64_t update:1; - uint64_t shift:1; - uint64_t capture:1; -#endif - } cn56xx; - struct cvmx_ciu_qlm_jtgd_cn56xxp1 { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t capture:1; - uint64_t shift:1; - uint64_t update:1; - uint64_t reserved_37_60:24; - uint64_t shft_cnt:5; - uint64_t shft_reg:32; -#else - uint64_t shft_reg:32; - uint64_t shft_cnt:5; - uint64_t reserved_37_60:24; - uint64_t update:1; - uint64_t shift:1; - uint64_t capture:1; -#endif - } cn56xxp1; - struct cvmx_ciu_qlm_jtgd_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t capture:1; - uint64_t shift:1; - uint64_t update:1; - uint64_t reserved_43_60:18; - uint64_t select:3; - uint64_t reserved_37_39:3; - uint64_t shft_cnt:5; - uint64_t shft_reg:32; -#else - uint64_t shft_reg:32; - uint64_t shft_cnt:5; - uint64_t reserved_37_39:3; - uint64_t select:3; - uint64_t reserved_43_60:18; - uint64_t update:1; - uint64_t shift:1; - uint64_t capture:1; -#endif - } cn61xx; - struct cvmx_ciu_qlm_jtgd_cn61xx cn63xx; - struct cvmx_ciu_qlm_jtgd_cn61xx cn63xxp1; - struct cvmx_ciu_qlm_jtgd_cn61xx cn66xx; - struct cvmx_ciu_qlm_jtgd_s cn68xx; - struct cvmx_ciu_qlm_jtgd_s cn68xxp1; - struct cvmx_ciu_qlm_jtgd_cn61xx cnf71xx; -}; - -union cvmx_ciu_soft_bist { - uint64_t u64; - struct cvmx_ciu_soft_bist_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_bist:1; -#else - uint64_t soft_bist:1; - uint64_t reserved_1_63:63; -#endif + __BITFIELD_FIELD(uint64_t capture:1, + __BITFIELD_FIELD(uint64_t shift:1, + __BITFIELD_FIELD(uint64_t update:1, + __BITFIELD_FIELD(uint64_t reserved_45_60:16, + __BITFIELD_FIELD(uint64_t select:5, + __BITFIELD_FIELD(uint64_t reserved_37_39:3, + __BITFIELD_FIELD(uint64_t shft_cnt:5, + __BITFIELD_FIELD(uint64_t shft_reg:32, + ;)))))))) } s; - struct cvmx_ciu_soft_bist_s cn30xx; - struct cvmx_ciu_soft_bist_s cn31xx; - struct cvmx_ciu_soft_bist_s cn38xx; - struct cvmx_ciu_soft_bist_s cn38xxp2; - struct cvmx_ciu_soft_bist_s cn50xx; - struct cvmx_ciu_soft_bist_s cn52xx; - struct cvmx_ciu_soft_bist_s cn52xxp1; - struct cvmx_ciu_soft_bist_s cn56xx; - struct cvmx_ciu_soft_bist_s cn56xxp1; - struct cvmx_ciu_soft_bist_s cn58xx; - struct cvmx_ciu_soft_bist_s cn58xxp1; - struct cvmx_ciu_soft_bist_s cn61xx; - struct cvmx_ciu_soft_bist_s cn63xx; - struct cvmx_ciu_soft_bist_s cn63xxp1; - struct cvmx_ciu_soft_bist_s cn66xx; - struct cvmx_ciu_soft_bist_s cn68xx; - struct cvmx_ciu_soft_bist_s cn68xxp1; - struct cvmx_ciu_soft_bist_s cnf71xx; }; union cvmx_ciu_soft_prst { uint64_t u64; struct cvmx_ciu_soft_prst_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_3_63:61; - uint64_t host64:1; - uint64_t npi:1; - uint64_t soft_prst:1; -#else - uint64_t soft_prst:1; - uint64_t npi:1; - uint64_t host64:1; - uint64_t reserved_3_63:61; -#endif - } s; - struct cvmx_ciu_soft_prst_s cn30xx; - struct cvmx_ciu_soft_prst_s cn31xx; - struct cvmx_ciu_soft_prst_s cn38xx; - struct cvmx_ciu_soft_prst_s cn38xxp2; - struct cvmx_ciu_soft_prst_s cn50xx; - struct cvmx_ciu_soft_prst_cn52xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_prst:1; -#else - uint64_t soft_prst:1; - uint64_t reserved_1_63:63; -#endif - } cn52xx; - struct cvmx_ciu_soft_prst_cn52xx cn52xxp1; - struct cvmx_ciu_soft_prst_cn52xx cn56xx; - struct cvmx_ciu_soft_prst_cn52xx cn56xxp1; - struct cvmx_ciu_soft_prst_s cn58xx; - struct cvmx_ciu_soft_prst_s cn58xxp1; - struct cvmx_ciu_soft_prst_cn52xx cn61xx; - struct cvmx_ciu_soft_prst_cn52xx cn63xx; - struct cvmx_ciu_soft_prst_cn52xx cn63xxp1; - struct cvmx_ciu_soft_prst_cn52xx cn66xx; - struct cvmx_ciu_soft_prst_cn52xx cn68xx; - struct cvmx_ciu_soft_prst_cn52xx cn68xxp1; - struct cvmx_ciu_soft_prst_cn52xx cnf71xx; -}; - -union cvmx_ciu_soft_prst1 { - uint64_t u64; - struct cvmx_ciu_soft_prst1_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_prst:1; -#else - uint64_t soft_prst:1; - uint64_t reserved_1_63:63; -#endif - } s; - struct cvmx_ciu_soft_prst1_s cn52xx; - struct cvmx_ciu_soft_prst1_s cn52xxp1; - struct cvmx_ciu_soft_prst1_s cn56xx; - struct cvmx_ciu_soft_prst1_s cn56xxp1; - struct cvmx_ciu_soft_prst1_s cn61xx; - struct cvmx_ciu_soft_prst1_s cn63xx; - struct cvmx_ciu_soft_prst1_s cn63xxp1; - struct cvmx_ciu_soft_prst1_s cn66xx; - struct cvmx_ciu_soft_prst1_s cn68xx; - struct cvmx_ciu_soft_prst1_s cn68xxp1; - struct cvmx_ciu_soft_prst1_s cnf71xx; -}; - -union cvmx_ciu_soft_prst2 { - uint64_t u64; - struct cvmx_ciu_soft_prst2_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_prst:1; -#else - uint64_t soft_prst:1; - uint64_t reserved_1_63:63; -#endif - } s; - struct cvmx_ciu_soft_prst2_s cn66xx; -}; - -union cvmx_ciu_soft_prst3 { - uint64_t u64; - struct cvmx_ciu_soft_prst3_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_prst:1; -#else - uint64_t soft_prst:1; - uint64_t reserved_1_63:63; -#endif - } s; - struct cvmx_ciu_soft_prst3_s cn66xx; -}; - -union cvmx_ciu_soft_rst { - uint64_t u64; - struct cvmx_ciu_soft_rst_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t soft_rst:1; -#else - uint64_t soft_rst:1; - uint64_t reserved_1_63:63; -#endif - } s; - struct cvmx_ciu_soft_rst_s cn30xx; - struct cvmx_ciu_soft_rst_s cn31xx; - struct cvmx_ciu_soft_rst_s cn38xx; - struct cvmx_ciu_soft_rst_s cn38xxp2; - struct cvmx_ciu_soft_rst_s cn50xx; - struct cvmx_ciu_soft_rst_s cn52xx; - struct cvmx_ciu_soft_rst_s cn52xxp1; - struct cvmx_ciu_soft_rst_s cn56xx; - struct cvmx_ciu_soft_rst_s cn56xxp1; - struct cvmx_ciu_soft_rst_s cn58xx; - struct cvmx_ciu_soft_rst_s cn58xxp1; - struct cvmx_ciu_soft_rst_s cn61xx; - struct cvmx_ciu_soft_rst_s cn63xx; - struct cvmx_ciu_soft_rst_s cn63xxp1; - struct cvmx_ciu_soft_rst_s cn66xx; - struct cvmx_ciu_soft_rst_s cn68xx; - struct cvmx_ciu_soft_rst_s cn68xxp1; - struct cvmx_ciu_soft_rst_s cnf71xx; -}; - -union cvmx_ciu_sum1_iox_int { - uint64_t u64; - struct cvmx_ciu_sum1_iox_int_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif + __BITFIELD_FIELD(uint64_t reserved_3_63:61, + __BITFIELD_FIELD(uint64_t host64:1, + __BITFIELD_FIELD(uint64_t npi:1, + __BITFIELD_FIELD(uint64_t soft_prst:1, + ;)))) } s; - struct cvmx_ciu_sum1_iox_int_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_sum1_iox_int_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_sum1_iox_int_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_sum1_ppx_ip2 { - uint64_t u64; - struct cvmx_ciu_sum1_ppx_ip2_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_sum1_ppx_ip2_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_sum1_ppx_ip2_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_sum1_ppx_ip2_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_sum1_ppx_ip3 { - uint64_t u64; - struct cvmx_ciu_sum1_ppx_ip3_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_sum1_ppx_ip3_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_sum1_ppx_ip3_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_sum1_ppx_ip3_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_sum1_ppx_ip4 { - uint64_t u64; - struct cvmx_ciu_sum1_ppx_ip4_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } s; - struct cvmx_ciu_sum1_ppx_ip4_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_41_45:5; - uint64_t dpi_dma:1; - uint64_t reserved_38_39:2; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_4_17:14; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_17:14; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_39:2; - uint64_t dpi_dma:1; - uint64_t reserved_41_45:5; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cn61xx; - struct cvmx_ciu_sum1_ppx_ip4_cn66xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_62_62:1; - uint64_t srio3:1; - uint64_t srio2:1; - uint64_t reserved_57_59:3; - uint64_t dfm:1; - uint64_t reserved_53_55:3; - uint64_t lmc0:1; - uint64_t reserved_51_51:1; - uint64_t srio0:1; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t agl:1; - uint64_t reserved_38_45:8; - uint64_t agx1:1; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t dfa:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t zip:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t mii1:1; - uint64_t reserved_10_17:8; - uint64_t wdog:10; -#else - uint64_t wdog:10; - uint64_t reserved_10_17:8; - uint64_t mii1:1; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t zip:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t dfa:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t agx1:1; - uint64_t reserved_38_45:8; - uint64_t agl:1; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t srio0:1; - uint64_t reserved_51_51:1; - uint64_t lmc0:1; - uint64_t reserved_53_55:3; - uint64_t dfm:1; - uint64_t reserved_57_59:3; - uint64_t srio2:1; - uint64_t srio3:1; - uint64_t reserved_62_62:1; - uint64_t rst:1; -#endif - } cn66xx; - struct cvmx_ciu_sum1_ppx_ip4_cnf71xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t rst:1; - uint64_t reserved_53_62:10; - uint64_t lmc0:1; - uint64_t reserved_50_51:2; - uint64_t pem1:1; - uint64_t pem0:1; - uint64_t ptp:1; - uint64_t reserved_41_46:6; - uint64_t dpi_dma:1; - uint64_t reserved_37_39:3; - uint64_t agx0:1; - uint64_t dpi:1; - uint64_t sli:1; - uint64_t usb:1; - uint64_t reserved_32_32:1; - uint64_t key:1; - uint64_t rad:1; - uint64_t tim:1; - uint64_t reserved_28_28:1; - uint64_t pko:1; - uint64_t pip:1; - uint64_t ipd:1; - uint64_t l2c:1; - uint64_t pow:1; - uint64_t fpa:1; - uint64_t iob:1; - uint64_t mio:1; - uint64_t nand:1; - uint64_t reserved_4_18:15; - uint64_t wdog:4; -#else - uint64_t wdog:4; - uint64_t reserved_4_18:15; - uint64_t nand:1; - uint64_t mio:1; - uint64_t iob:1; - uint64_t fpa:1; - uint64_t pow:1; - uint64_t l2c:1; - uint64_t ipd:1; - uint64_t pip:1; - uint64_t pko:1; - uint64_t reserved_28_28:1; - uint64_t tim:1; - uint64_t rad:1; - uint64_t key:1; - uint64_t reserved_32_32:1; - uint64_t usb:1; - uint64_t sli:1; - uint64_t dpi:1; - uint64_t agx0:1; - uint64_t reserved_37_39:3; - uint64_t dpi_dma:1; - uint64_t reserved_41_46:6; - uint64_t ptp:1; - uint64_t pem0:1; - uint64_t pem1:1; - uint64_t reserved_50_51:2; - uint64_t lmc0:1; - uint64_t reserved_53_62:10; - uint64_t rst:1; -#endif - } cnf71xx; -}; - -union cvmx_ciu_sum2_iox_int { - uint64_t u64; - struct cvmx_ciu_sum2_iox_int_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_sum2_iox_int_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_sum2_iox_int_cn61xx cn66xx; - struct cvmx_ciu_sum2_iox_int_s cnf71xx; -}; - -union cvmx_ciu_sum2_ppx_ip2 { - uint64_t u64; - struct cvmx_ciu_sum2_ppx_ip2_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_sum2_ppx_ip2_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_sum2_ppx_ip2_cn61xx cn66xx; - struct cvmx_ciu_sum2_ppx_ip2_s cnf71xx; -}; - -union cvmx_ciu_sum2_ppx_ip3 { - uint64_t u64; - struct cvmx_ciu_sum2_ppx_ip3_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_sum2_ppx_ip3_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_sum2_ppx_ip3_cn61xx cn66xx; - struct cvmx_ciu_sum2_ppx_ip3_s cnf71xx; -}; - -union cvmx_ciu_sum2_ppx_ip4 { - uint64_t u64; - struct cvmx_ciu_sum2_ppx_ip4_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_15_63:49; - uint64_t endor:2; - uint64_t eoi:1; - uint64_t reserved_10_11:2; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_11:2; - uint64_t eoi:1; - uint64_t endor:2; - uint64_t reserved_15_63:49; -#endif - } s; - struct cvmx_ciu_sum2_ppx_ip4_cn61xx { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_10_63:54; - uint64_t timer:6; - uint64_t reserved_0_3:4; -#else - uint64_t reserved_0_3:4; - uint64_t timer:6; - uint64_t reserved_10_63:54; -#endif - } cn61xx; - struct cvmx_ciu_sum2_ppx_ip4_cn61xx cn66xx; - struct cvmx_ciu_sum2_ppx_ip4_s cnf71xx; }; union cvmx_ciu_timx { uint64_t u64; struct cvmx_ciu_timx_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_37_63:27; - uint64_t one_shot:1; - uint64_t len:36; -#else - uint64_t len:36; - uint64_t one_shot:1; - uint64_t reserved_37_63:27; -#endif - } s; - struct cvmx_ciu_timx_s cn30xx; - struct cvmx_ciu_timx_s cn31xx; - struct cvmx_ciu_timx_s cn38xx; - struct cvmx_ciu_timx_s cn38xxp2; - struct cvmx_ciu_timx_s cn50xx; - struct cvmx_ciu_timx_s cn52xx; - struct cvmx_ciu_timx_s cn52xxp1; - struct cvmx_ciu_timx_s cn56xx; - struct cvmx_ciu_timx_s cn56xxp1; - struct cvmx_ciu_timx_s cn58xx; - struct cvmx_ciu_timx_s cn58xxp1; - struct cvmx_ciu_timx_s cn61xx; - struct cvmx_ciu_timx_s cn63xx; - struct cvmx_ciu_timx_s cn63xxp1; - struct cvmx_ciu_timx_s cn66xx; - struct cvmx_ciu_timx_s cn68xx; - struct cvmx_ciu_timx_s cn68xxp1; - struct cvmx_ciu_timx_s cnf71xx; -}; - -union cvmx_ciu_tim_multi_cast { - uint64_t u64; - struct cvmx_ciu_tim_multi_cast_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_1_63:63; - uint64_t en:1; -#else - uint64_t en:1; - uint64_t reserved_1_63:63; -#endif + __BITFIELD_FIELD(uint64_t reserved_37_63:27, + __BITFIELD_FIELD(uint64_t one_shot:1, + __BITFIELD_FIELD(uint64_t len:36, + ;))) } s; - struct cvmx_ciu_tim_multi_cast_s cn61xx; - struct cvmx_ciu_tim_multi_cast_s cn66xx; - struct cvmx_ciu_tim_multi_cast_s cnf71xx; }; union cvmx_ciu_wdogx { uint64_t u64; struct cvmx_ciu_wdogx_s { -#ifdef __BIG_ENDIAN_BITFIELD - uint64_t reserved_46_63:18; - uint64_t gstopen:1; - uint64_t dstop:1; - uint64_t cnt:24; - uint64_t len:16; - uint64_t state:2; - uint64_t mode:2; -#else - uint64_t mode:2; - uint64_t state:2; - uint64_t len:16; - uint64_t cnt:24; - uint64_t dstop:1; - uint64_t gstopen:1; - uint64_t reserved_46_63:18; -#endif + __BITFIELD_FIELD(uint64_t reserved_46_63:18, + __BITFIELD_FIELD(uint64_t gstopen:1, + __BITFIELD_FIELD(uint64_t dstop:1, + __BITFIELD_FIELD(uint64_t cnt:24, + __BITFIELD_FIELD(uint64_t len:16, + __BITFIELD_FIELD(uint64_t state:2, + __BITFIELD_FIELD(uint64_t mode:2, + ;))))))) } s; - struct cvmx_ciu_wdogx_s cn30xx; - struct cvmx_ciu_wdogx_s cn31xx; - struct cvmx_ciu_wdogx_s cn38xx; - struct cvmx_ciu_wdogx_s cn38xxp2; - struct cvmx_ciu_wdogx_s cn50xx; - struct cvmx_ciu_wdogx_s cn52xx; - struct cvmx_ciu_wdogx_s cn52xxp1; - struct cvmx_ciu_wdogx_s cn56xx; - struct cvmx_ciu_wdogx_s cn56xxp1; - struct cvmx_ciu_wdogx_s cn58xx; - struct cvmx_ciu_wdogx_s cn58xxp1; - struct cvmx_ciu_wdogx_s cn61xx; - struct cvmx_ciu_wdogx_s cn63xx; - struct cvmx_ciu_wdogx_s cn63xxp1; - struct cvmx_ciu_wdogx_s cn66xx; - struct cvmx_ciu_wdogx_s cn68xx; - struct cvmx_ciu_wdogx_s cn68xxp1; - struct cvmx_ciu_wdogx_s cnf71xx; }; -#endif +#endif /* __CVMX_CIU_DEFS_H__ */ diff --git a/arch/mips/include/asm/octeon/cvmx-gmxx-defs.h b/arch/mips/include/asm/octeon/cvmx-gmxx-defs.h index e347496a33c3..80e4f8358b81 100644 --- a/arch/mips/include/asm/octeon/cvmx-gmxx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-gmxx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -2070,6 +2070,8 @@ static inline uint64_t CVMX_GMXX_XAUI_EXT_LOOPBACK(unsigned long block_id) return CVMX_ADD_IO_SEG(0x0001180008000540ull) + (block_id) * 0x8000000ull; } +void __cvmx_interrupt_gmxx_enable(int interface); + union cvmx_gmxx_bad_reg { uint64_t u64; struct cvmx_gmxx_bad_reg_s { diff --git a/arch/mips/include/asm/octeon/cvmx-pcsx-defs.h b/arch/mips/include/asm/octeon/cvmx-pcsx-defs.h index a5e8fd861c37..39da7f9d7b3f 100644 --- a/arch/mips/include/asm/octeon/cvmx-pcsx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-pcsx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -334,6 +334,8 @@ static inline uint64_t CVMX_PCSX_TX_RXX_POLARITY_REG(unsigned long offset, unsig return CVMX_ADD_IO_SEG(0x00011800B0001048ull) + ((offset) + (block_id) * 0x20000ull) * 1024; } +void __cvmx_interrupt_pcsx_intx_en_reg_enable(int index, int block); + union cvmx_pcsx_anx_adv_reg { uint64_t u64; struct cvmx_pcsx_anx_adv_reg_s { diff --git a/arch/mips/include/asm/octeon/cvmx-pcsxx-defs.h b/arch/mips/include/asm/octeon/cvmx-pcsxx-defs.h index b5b45d26f1c5..847dd9dca6ea 100644 --- a/arch/mips/include/asm/octeon/cvmx-pcsxx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-pcsxx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -268,6 +268,8 @@ static inline uint64_t CVMX_PCSXX_TX_RX_STATES_REG(unsigned long block_id) return CVMX_ADD_IO_SEG(0x00011800B0000830ull) + (block_id) * 0x1000000ull; } +void __cvmx_interrupt_pcsxx_int_en_reg_enable(int index); + union cvmx_pcsxx_10gbx_status_reg { uint64_t u64; struct cvmx_pcsxx_10gbx_status_reg_s { diff --git a/arch/mips/include/asm/octeon/cvmx-spxx-defs.h b/arch/mips/include/asm/octeon/cvmx-spxx-defs.h index c7d601d9446e..f4c4e8051160 100644 --- a/arch/mips/include/asm/octeon/cvmx-spxx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-spxx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -45,6 +45,8 @@ #define CVMX_SPXX_TPA_SEL(block_id) (CVMX_ADD_IO_SEG(0x0001180090000328ull) + ((block_id) & 1) * 0x8000000ull) #define CVMX_SPXX_TRN4_CTL(block_id) (CVMX_ADD_IO_SEG(0x0001180090000360ull) + ((block_id) & 1) * 0x8000000ull) +void __cvmx_interrupt_spxx_int_msk_enable(int index); + union cvmx_spxx_bckprs_cnt { uint64_t u64; struct cvmx_spxx_bckprs_cnt_s { diff --git a/arch/mips/include/asm/octeon/cvmx-stxx-defs.h b/arch/mips/include/asm/octeon/cvmx-stxx-defs.h index 146354005d3b..3c409a854d91 100644 --- a/arch/mips/include/asm/octeon/cvmx-stxx-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-stxx-defs.h @@ -4,7 +4,7 @@ * Contact: support@caviumnetworks.com * This file is part of the OCTEON SDK * - * Copyright (c) 2003-2012 Cavium Networks + * Copyright (C) 2003-2018 Cavium, Inc. * * This file is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, Version 2, as @@ -45,6 +45,8 @@ #define CVMX_STXX_STAT_CTL(block_id) (CVMX_ADD_IO_SEG(0x0001180090000638ull) + ((block_id) & 1) * 0x8000000ull) #define CVMX_STXX_STAT_PKT_XMT(block_id) (CVMX_ADD_IO_SEG(0x0001180090000640ull) + ((block_id) & 1) * 0x8000000ull) +void __cvmx_interrupt_stxx_int_msk_enable(int index); + union cvmx_stxx_arb_ctl { uint64_t u64; struct cvmx_stxx_arb_ctl_s { diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h index c99c4b6a79f4..60481502826a 100644 --- a/arch/mips/include/asm/octeon/octeon.h +++ b/arch/mips/include/asm/octeon/octeon.h @@ -279,13 +279,12 @@ union octeon_cvmemctl { } s; }; -extern void octeon_write_lcd(const char *s); extern void octeon_check_cpu_bist(void); -extern int octeon_get_boot_uart(void); -struct uart_port; -extern unsigned int octeon_serial_in(struct uart_port *, int); -extern void octeon_serial_out(struct uart_port *, int, int); +int octeon_prune_device_tree(void); +extern const char __appended_dtb; +extern const char __dtb_octeon_3xxx_begin; +extern const char __dtb_octeon_68xx_begin; /** * Write a 32bit value to the Octeon NPI register space diff --git a/arch/mips/include/asm/octeon/pci-octeon.h b/arch/mips/include/asm/octeon/pci-octeon.h index 1884609741a8..b12d9a3fbfb6 100644 --- a/arch/mips/include/asm/octeon/pci-octeon.h +++ b/arch/mips/include/asm/octeon/pci-octeon.h @@ -63,4 +63,7 @@ enum octeon_dma_bar_type { */ extern enum octeon_dma_bar_type octeon_dma_bar_type; +void octeon_pci_dma_init(void); +extern char *octeon_swiotlb; + #endif diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h index ad461216b5a1..e8cc328fce2d 100644 --- a/arch/mips/include/asm/page.h +++ b/arch/mips/include/asm/page.h @@ -80,7 +80,12 @@ extern void build_copy_page(void); * used in our early mem init code for all memory models. * So always define it. */ -#define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET) +#ifdef CONFIG_MIPS_AUTO_PFN_OFFSET +extern unsigned long ARCH_PFN_OFFSET; +# define ARCH_PFN_OFFSET ARCH_PFN_OFFSET +#else +# define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET) +#endif extern void clear_page(void * page); extern void copy_page(void * to, void * from); @@ -252,8 +257,8 @@ extern int __virt_addr_valid(const volatile void *kaddr); ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) -#define UNCAC_ADDR(addr) ((addr) - PAGE_OFFSET + UNCAC_BASE) -#define CAC_ADDR(addr) ((addr) - UNCAC_BASE + PAGE_OFFSET) +#define UNCAC_ADDR(addr) (UNCAC_BASE + __pa(addr)) +#define CAC_ADDR(addr) ((unsigned long)__va((addr) - UNCAC_BASE)) #include <asm-generic/memory_model.h> #include <asm-generic/getorder.h> diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index af34afbc32d9..b2fa62922d88 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -141,7 +141,7 @@ struct mips_fpu_struct { #define NUM_DSP_REGS 6 -typedef __u32 dspreg_t; +typedef unsigned long dspreg_t; struct mips_dsp_state { dspreg_t dspr[NUM_DSP_REGS]; @@ -386,7 +386,20 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[29]) #define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status) +#ifdef CONFIG_CPU_LOONGSON3 +/* + * Loongson-3's SFB (Store-Fill-Buffer) may buffer writes indefinitely when a + * tight read loop is executed, because reads take priority over writes & the + * hardware (incorrectly) doesn't ensure that writes will eventually occur. + * + * Since spin loops of any kind should have a cpu_relax() in them, force an SFB + * flush from cpu_relax() such that any pending writes will become visible as + * expected. + */ +#define cpu_relax() smp_mb() +#else #define cpu_relax() barrier() +#endif /* * Return_address is a replacement for __builtin_return_address(count) diff --git a/arch/mips/include/asm/setup.h b/arch/mips/include/asm/setup.h index d49d247d48a1..bb36a400203d 100644 --- a/arch/mips/include/asm/setup.h +++ b/arch/mips/include/asm/setup.h @@ -2,8 +2,10 @@ #ifndef _MIPS_SETUP_H #define _MIPS_SETUP_H +#include <linux/types.h> #include <uapi/asm/setup.h> +extern void prom_putchar(char); extern void setup_early_printk(void); #ifdef CONFIG_EARLY_PRINTK_8250 diff --git a/arch/mips/include/asm/sgialib.h b/arch/mips/include/asm/sgialib.h index 195db5045ae5..0d9fad5915fe 100644 --- a/arch/mips/include/asm/sgialib.h +++ b/arch/mips/include/asm/sgialib.h @@ -31,7 +31,6 @@ extern int prom_flags; #define PROM_FLAG_DONT_FREE_TEMP 4 /* Simple char-by-char console I/O. */ -extern void prom_putchar(char c); extern char prom_getchar(void); /* Get next memory descriptor after CURR, returns first descriptor diff --git a/arch/mips/include/asm/sim.h b/arch/mips/include/asm/sim.h index 91831800c480..59f31a95facd 100644 --- a/arch/mips/include/asm/sim.h +++ b/arch/mips/include/asm/sim.h @@ -39,8 +39,6 @@ __asm__( \ ".end\t__" #symbol "\n\t" \ ".size\t__" #symbol",. - __" #symbol) -#define nabi_no_regargs - #endif /* CONFIG_32BIT */ #ifdef CONFIG_64BIT @@ -67,16 +65,6 @@ __asm__( \ ".end\t__" #symbol "\n\t" \ ".size\t__" #symbol",. - __" #symbol) -#define nabi_no_regargs \ - unsigned long __dummy0, \ - unsigned long __dummy1, \ - unsigned long __dummy2, \ - unsigned long __dummy3, \ - unsigned long __dummy4, \ - unsigned long __dummy5, \ - unsigned long __dummy6, \ - unsigned long __dummy7, - #endif /* CONFIG_64BIT */ #endif /* _ASM_SIM_H */ diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 88ebd83b3bf9..056a6bf13491 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -25,7 +25,17 @@ extern cpumask_t cpu_sibling_map[]; extern cpumask_t cpu_core_map[]; extern cpumask_t cpu_foreign_map[]; -#define raw_smp_processor_id() (current_thread_info()->cpu) +static inline int raw_smp_processor_id(void) +{ +#if defined(__VDSO__) + extern int vdso_smp_processor_id(void) + __compiletime_error("VDSO should not call smp_processor_id()"); + return vdso_smp_processor_id(); +#else + return current_thread_info()->cpu; +#endif +} +#define raw_smp_processor_id raw_smp_processor_id /* Map from cpu id to sequential logical cpu number. This will only not be idempotent when cpus failed to come on-line. */ diff --git a/arch/mips/include/asm/txx9/generic.h b/arch/mips/include/asm/txx9/generic.h index 64887d3c7ec3..9a2c47bf3c40 100644 --- a/arch/mips/include/asm/txx9/generic.h +++ b/arch/mips/include/asm/txx9/generic.h @@ -49,7 +49,6 @@ void txx9_spi_init(int busid, unsigned long base, int irq); void txx9_ethaddr_init(unsigned int id, unsigned char *ethaddr); void txx9_sio_init(unsigned long baseaddr, int irq, unsigned int line, unsigned int sclk, int nocts); -void prom_putchar(char c); #ifdef CONFIG_EARLY_PRINTK extern void (*txx9_prom_putchar)(char c); void txx9_sio_putchar_init(unsigned long baseaddr); diff --git a/arch/mips/include/asm/txx9/tx4939.h b/arch/mips/include/asm/txx9/tx4939.h index 6d667087f2aa..00805ac6e9fc 100644 --- a/arch/mips/include/asm/txx9/tx4939.h +++ b/arch/mips/include/asm/txx9/tx4939.h @@ -101,13 +101,6 @@ struct tx4939_irc_reg { struct tx4939_le_reg maskext; }; -struct tx4939_rtc_reg { - __u32 ctl; - __u32 adr; - __u32 dat; - __u32 tbc; -}; - struct tx4939_crypto_reg { struct tx4939_le_reg csr; struct tx4939_le_reg idesptr; @@ -370,26 +363,6 @@ struct tx4939_vpc_desc { #define TX4939_CLKCTR_CYPRST 0x00000001 /* - * RTC - */ -#define TX4939_RTCCTL_ALME 0x00000080 -#define TX4939_RTCCTL_ALMD 0x00000040 -#define TX4939_RTCCTL_BUSY 0x00000020 - -#define TX4939_RTCCTL_COMMAND 0x00000007 -#define TX4939_RTCCTL_COMMAND_NOP 0x00000000 -#define TX4939_RTCCTL_COMMAND_GETTIME 0x00000001 -#define TX4939_RTCCTL_COMMAND_SETTIME 0x00000002 -#define TX4939_RTCCTL_COMMAND_GETALARM 0x00000003 -#define TX4939_RTCCTL_COMMAND_SETALARM 0x00000004 - -#define TX4939_RTCTBC_PM 0x00000080 -#define TX4939_RTCTBC_COMP 0x0000007f - -#define TX4939_RTC_REG_RAMSIZE 0x00000100 -#define TX4939_RTC_REG_RWBSIZE 0x00000006 - -/* * CRYPTO */ #define TX4939_CRYPTO_CSR_SAESO 0x08000000 @@ -498,8 +471,6 @@ struct tx4939_vpc_desc { #define tx4939_ccfgptr \ ((struct tx4939_ccfg_reg __iomem *)TX4939_CCFG_REG) #define tx4939_sramcptr tx4938_sramcptr -#define tx4939_rtcptr \ - ((struct tx4939_rtc_reg __iomem *)TX4939_RTC_REG) #define tx4939_cryptoptr \ ((struct tx4939_crypto_reg __iomem *)TX4939_CRYPTO_REG) #define tx4939_vpcptr ((struct tx4939_vpc_reg __iomem *)TX4939_VPC_REG) diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index d626a9a391cc..d31bc2f01208 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -16,6 +16,8 @@ #include <linux/bootmem.h> #include <linux/spinlock.h> #include <linux/gfp.h> +#include <linux/dma-direct.h> +#include <linux/dma-noncoherent.h> #include <asm/mipsregs.h> #include <asm/jazz.h> #include <asm/io.h> @@ -86,6 +88,7 @@ static int __init vdma_init(void) printk(KERN_INFO "VDMA: R4030 DMA pagetables initialized.\n"); return 0; } +arch_initcall(vdma_init); /* * Allocate DMA pagetables using a simple first-fit algorithm @@ -556,4 +559,140 @@ int vdma_get_enable(int channel) return enable; } -arch_initcall(vdma_init); +static void *jazz_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + void *ret; + + ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!ret) + return NULL; + + *dma_handle = vdma_alloc(virt_to_phys(ret), size); + if (*dma_handle == VDMA_ERROR) { + dma_direct_free(dev, size, ret, *dma_handle, attrs); + return NULL; + } + + if (!(attrs & DMA_ATTR_NON_CONSISTENT)) { + dma_cache_wback_inv((unsigned long)ret, size); + ret = (void *)UNCAC_ADDR(ret); + } + return ret; +} + +static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_handle, unsigned long attrs) +{ + vdma_free(dma_handle); + if (!(attrs & DMA_ATTR_NON_CONSISTENT)) + vaddr = (void *)CAC_ADDR((unsigned long)vaddr); + return dma_direct_free(dev, size, vaddr, dma_handle, attrs); +} + +static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + phys_addr_t phys = page_to_phys(page) + offset; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_device(dev, phys, size, dir); + return vdma_alloc(phys, size); +} + +static void jazz_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_cpu(dev, vdma_log2phys(dma_addr), size, dir); + vdma_free(dma_addr); +} + +static int jazz_dma_map_sg(struct device *dev, struct scatterlist *sglist, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + int i; + struct scatterlist *sg; + + for_each_sg(sglist, sg, nents, i) { + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, + dir); + sg->dma_address = vdma_alloc(sg_phys(sg), sg->length); + if (sg->dma_address == VDMA_ERROR) + return 0; + sg_dma_len(sg) = sg->length; + } + + return nents; +} + +static void jazz_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + int i; + struct scatterlist *sg; + + for_each_sg(sglist, sg, nents, i) { + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, + dir); + vdma_free(sg->dma_address); + } +} + +static void jazz_dma_sync_single_for_device(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ + arch_sync_dma_for_device(dev, vdma_log2phys(addr), size, dir); +} + +static void jazz_dma_sync_single_for_cpu(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ + arch_sync_dma_for_cpu(dev, vdma_log2phys(addr), size, dir); +} + +static void jazz_dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) + arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir); +} + +static void jazz_dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) + arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); +} + +static int jazz_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_addr == VDMA_ERROR; +} + +const struct dma_map_ops jazz_dma_ops = { + .alloc = jazz_dma_alloc, + .free = jazz_dma_free, + .mmap = arch_dma_mmap, + .map_page = jazz_dma_map_page, + .unmap_page = jazz_dma_unmap_page, + .map_sg = jazz_dma_map_sg, + .unmap_sg = jazz_dma_unmap_sg, + .sync_single_for_cpu = jazz_dma_sync_single_for_cpu, + .sync_single_for_device = jazz_dma_sync_single_for_device, + .sync_sg_for_cpu = jazz_dma_sync_sg_for_cpu, + .sync_sg_for_device = jazz_dma_sync_sg_for_device, + .dma_supported = dma_direct_supported, + .cache_sync = arch_dma_cache_sync, + .mapping_error = jazz_dma_mapping_error, +}; +EXPORT_SYMBOL(jazz_dma_ops); diff --git a/arch/mips/jazz/setup.c b/arch/mips/jazz/setup.c index 448fd41792e4..1b5e121c3f0d 100644 --- a/arch/mips/jazz/setup.c +++ b/arch/mips/jazz/setup.c @@ -16,6 +16,7 @@ #include <linux/screen_info.h> #include <linux/platform_device.h> #include <linux/serial_8250.h> +#include <linux/dma-mapping.h> #include <asm/jazz.h> #include <asm/jazzdma.h> @@ -136,10 +137,16 @@ static struct resource jazz_esp_rsrc[] = { } }; +static u64 jazz_esp_dma_mask = DMA_BIT_MASK(32); + static struct platform_device jazz_esp_pdev = { .name = "jazz_esp", .num_resources = ARRAY_SIZE(jazz_esp_rsrc), - .resource = jazz_esp_rsrc + .resource = jazz_esp_rsrc, + .dev = { + .dma_mask = &jazz_esp_dma_mask, + .coherent_dma_mask = DMA_BIT_MASK(32), + } }; static struct resource jazz_sonic_rsrc[] = { @@ -155,10 +162,16 @@ static struct resource jazz_sonic_rsrc[] = { } }; +static u64 jazz_sonic_dma_mask = DMA_BIT_MASK(32); + static struct platform_device jazz_sonic_pdev = { .name = "jazzsonic", .num_resources = ARRAY_SIZE(jazz_sonic_rsrc), - .resource = jazz_sonic_rsrc + .resource = jazz_sonic_rsrc, + .dev = { + .dma_mask = &jazz_sonic_dma_mask, + .coherent_dma_mask = DMA_BIT_MASK(32), + } }; static struct resource jazz_cmos_rsrc[] = { diff --git a/arch/mips/jz4740/Platform b/arch/mips/jz4740/Platform index 28448d358c10..a2a5a85ea1f9 100644 --- a/arch/mips/jz4740/Platform +++ b/arch/mips/jz4740/Platform @@ -1,4 +1,4 @@ platform-$(CONFIG_MACH_INGENIC) += jz4740/ cflags-$(CONFIG_MACH_INGENIC) += -I$(srctree)/arch/mips/include/asm/mach-jz4740 load-$(CONFIG_MACH_INGENIC) += 0xffffffff80010000 -zload-$(CONFIG_MACH_INGENIC) += 0xffffffff80600000 +zload-$(CONFIG_MACH_INGENIC) += 0xffffffff81000000 diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index b2509c19cfb5..d535fc706a8b 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1849,7 +1849,8 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) set_elf_platform(cpu, "loongson3a"); set_isa(c, MIPS_CPU_ISA_M64R2); break; - case PRID_REV_LOONGSON3A_R3: + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: c->cputype = CPU_LOONGSON3; __cpu_name[cpu] = "ICT Loongson-3"; set_elf_platform(cpu, "loongson3a"); diff --git a/arch/mips/kernel/early_printk.c b/arch/mips/kernel/early_printk.c index 505cb77d1280..4a1647ddfbd9 100644 --- a/arch/mips/kernel/early_printk.c +++ b/arch/mips/kernel/early_printk.c @@ -14,8 +14,6 @@ #include <asm/setup.h> -extern void prom_putchar(char); - static void early_console_write(struct console *con, const char *s, unsigned n) { while (n-- && *s) { diff --git a/arch/mips/kernel/early_printk_8250.c b/arch/mips/kernel/early_printk_8250.c index 83cea3767556..ea26614afac6 100644 --- a/arch/mips/kernel/early_printk_8250.c +++ b/arch/mips/kernel/early_printk_8250.c @@ -20,6 +20,7 @@ #include <linux/io.h> #include <linux/serial_core.h> #include <linux/serial_reg.h> +#include <asm/setup.h> static void __iomem *serial8250_base; static unsigned int serial8250_reg_shift; diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 37b9383eacd3..6c257b52f57f 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -354,16 +354,56 @@ NESTED(ejtag_debug_handler, PT_SIZE, sp) sll k0, k0, 30 # Check for SDBBP. bgez k0, ejtag_return +#ifdef CONFIG_SMP +1: PTR_LA k0, ejtag_debug_buffer_spinlock + ll k0, 0(k0) + bnez k0, 1b + PTR_LA k0, ejtag_debug_buffer_spinlock + sc k0, 0(k0) + beqz k0, 1b +# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC + sync +# endif + + PTR_LA k0, ejtag_debug_buffer + LONG_S k1, 0(k0) + + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + + PTR_LA k1, ejtag_debug_buffer + LONG_L k1, 0(k1) + LONG_S k1, 0(k0) + + PTR_LA k0, ejtag_debug_buffer_spinlock + sw zero, 0(k0) +#else PTR_LA k0, ejtag_debug_buffer LONG_S k1, 0(k0) +#endif + SAVE_ALL move a0, sp jal ejtag_exception_handler RESTORE_ALL + +#ifdef CONFIG_SMP + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + LONG_L k1, 0(k0) +#else PTR_LA k0, ejtag_debug_buffer LONG_L k1, 0(k0) +#endif ejtag_return: + back_to_back_c0_hazard MFC0 k0, CP0_DESAVE .set mips32 deret @@ -377,6 +417,12 @@ ejtag_return: .data EXPORT(ejtag_debug_buffer) .fill LONGSIZE +#ifdef CONFIG_SMP +EXPORT(ejtag_debug_buffer_spinlock) + .fill LONGSIZE +EXPORT(ejtag_debug_buffer_per_cpu) + .fill LONGSIZE * NR_CPUS +#endif .previous __INIT diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index 7c246b69c545..046846999efd 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -33,21 +33,21 @@ void (*cpu_wait)(void); EXPORT_SYMBOL(cpu_wait); -static void r3081_wait(void) +static void __cpuidle r3081_wait(void) { unsigned long cfg = read_c0_conf(); write_c0_conf(cfg | R30XX_CONF_HALT); local_irq_enable(); } -static void r39xx_wait(void) +static void __cpuidle r39xx_wait(void) { if (!need_resched()) write_c0_conf(read_c0_conf() | TX39_CONF_HALT); local_irq_enable(); } -void r4k_wait(void) +void __cpuidle r4k_wait(void) { local_irq_enable(); __r4k_wait(); @@ -60,7 +60,7 @@ void r4k_wait(void) * interrupt is requested" restriction in the MIPS32/MIPS64 architecture makes * using this version a gamble. */ -void r4k_wait_irqoff(void) +void __cpuidle r4k_wait_irqoff(void) { if (!need_resched()) __asm__( @@ -75,7 +75,7 @@ void r4k_wait_irqoff(void) * The RM7000 variant has to handle erratum 38. The workaround is to not * have any pending stores when the WAIT instruction is executed. */ -static void rm7k_wait_irqoff(void) +static void __cpuidle rm7k_wait_irqoff(void) { if (!need_resched()) __asm__( @@ -96,7 +96,7 @@ static void rm7k_wait_irqoff(void) * since coreclock (and the cp0 counter) stops upon executing it. Only an * interrupt can wake it, so they must be enabled before entering idle modes. */ -static void au1k_wait(void) +static void __cpuidle au1k_wait(void) { unsigned long c0status = read_c0_status() | 1; /* irqs on */ diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index f5c8bce70db2..54cd675c5d1d 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -326,19 +326,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) preempt_enable_no_resched(); } return 1; - } else { - if (addr->word != breakpoint_insn.word) { - /* - * The breakpoint instruction was removed by - * another cpu right after we hit, no further - * handling of this interrupt is appropriate - */ - ret = 1; - goto no_kprobe; - } - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) - goto ss_probe; + } else if (addr->word != breakpoint_insn.word) { + /* + * The breakpoint instruction was removed by + * another cpu right after we hit, no further + * handling of this interrupt is appropriate + */ + ret = 1; } goto no_kprobe; } @@ -364,10 +358,11 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) if (p->pre_handler && p->pre_handler(p, regs)) { /* handler has already set things up, so skip ss setup */ + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; } -ss_probe: prepare_singlestep(p, regs, kcb); if (kcb->flags & SKIP_DELAYSLOT) { kcb->kprobe_status = KPROBE_HIT_SSDONE; @@ -468,51 +463,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, return ret; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - kcb->jprobe_saved_regs = *regs; - kcb->jprobe_saved_sp = regs->regs[29]; - - memcpy(kcb->jprobes_stack, (void *)kcb->jprobe_saved_sp, - MIN_JPROBES_STACK_SIZE(kcb->jprobe_saved_sp)); - - regs->cp0_epc = (unsigned long)(jp->entry); - - return 1; -} - -/* Defined in the inline asm below. */ -void jprobe_return_end(void); - -void __kprobes jprobe_return(void) -{ - /* Assembler quirk necessitates this '0,code' business. */ - asm volatile( - "break 0,%0\n\t" - ".globl jprobe_return_end\n" - "jprobe_return_end:\n" - : : "n" (BRK_KPROBE_BP) : "memory"); -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - if (regs->cp0_epc >= (unsigned long)jprobe_return && - regs->cp0_epc <= (unsigned long)jprobe_return_end) { - *regs = kcb->jprobe_saved_regs; - memcpy((void *)kcb->jprobe_saved_sp, kcb->jprobes_stack, - MIN_JPROBES_STACK_SIZE(kcb->jprobe_saved_sp)); - preempt_enable_no_resched(); - - return 1; - } - return 0; -} - /* * Function return probe trampoline: * - init_kprobes() establishes a probepoint here @@ -595,9 +545,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, kretprobe_assert(ri, orig_ret_address, trampoline_address); instruction_pointer(regs) = orig_ret_address; - reset_current_kprobe(); kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c index 318f1c05c5b3..6b61be486303 100644 --- a/arch/mips/kernel/linux32.c +++ b/arch/mips/kernel/linux32.c @@ -43,17 +43,6 @@ #include <asm/mmu_context.h> #include <asm/mman.h> -/* Use this to get at 32-bit user passed pointers. */ -/* A() macro should be used for places where you e.g. - have some internal variable u32 and just want to get - rid of a compiler warning. AA() has to be used in - places where you want to convert a function argument - to 32bit pointer or when you e.g. access pt_regs - structure and want to consider 32bit registers only. - */ -#define A(__x) ((unsigned long)(__x)) -#define AA(__x) ((unsigned long)((int)__x)) - #ifdef __MIPSEB__ #define merge_64(r1, r2) ((((r1) & 0xffffffffUL) << 32) + ((r2) & 0xffffffffUL)) #endif @@ -61,24 +50,6 @@ #define merge_64(r1, r2) ((((r2) & 0xffffffffUL) << 32) + ((r1) & 0xffffffffUL)) #endif -SYSCALL_DEFINE6(32_mmap2, unsigned long, addr, unsigned long, len, - unsigned long, prot, unsigned long, flags, unsigned long, fd, - unsigned long, pgoff) -{ - if (pgoff & (~PAGE_MASK >> 12)) - return -EINVAL; - return ksys_mmap_pgoff(addr, len, prot, flags, fd, - pgoff >> (PAGE_SHIFT-12)); -} - -#define RLIM_INFINITY32 0x7fffffff -#define RESOURCE32(x) ((x > RLIM_INFINITY32) ? RLIM_INFINITY32 : x) - -struct rlimit32 { - int rlim_cur; - int rlim_max; -}; - SYSCALL_DEFINE4(32_truncate64, const char __user *, path, unsigned long, __dummy, unsigned long, a2, unsigned long, a3) { diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 8d85046adcc8..8fc69891e117 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -29,6 +29,8 @@ #include <linux/kallsyms.h> #include <linux/random.h> #include <linux/prctl.h> +#include <linux/nmi.h> +#include <linux/cpu.h> #include <asm/asm.h> #include <asm/bootinfo.h> @@ -655,28 +657,42 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ALMASK; } -static void arch_dump_stack(void *info) +static DEFINE_PER_CPU(call_single_data_t, backtrace_csd); +static struct cpumask backtrace_csd_busy; + +static void handle_backtrace(void *info) { - struct pt_regs *regs; + nmi_cpu_backtrace(get_irq_regs()); + cpumask_clear_cpu(smp_processor_id(), &backtrace_csd_busy); +} - regs = get_irq_regs(); +static void raise_backtrace(cpumask_t *mask) +{ + call_single_data_t *csd; + int cpu; - if (regs) - show_regs(regs); + for_each_cpu(cpu, mask) { + /* + * If we previously sent an IPI to the target CPU & it hasn't + * cleared its bit in the busy cpumask then it didn't handle + * our previous IPI & it's not safe for us to reuse the + * call_single_data_t. + */ + if (cpumask_test_and_set_cpu(cpu, &backtrace_csd_busy)) { + pr_warn("Unable to send backtrace IPI to CPU%u - perhaps it hung?\n", + cpu); + continue; + } - dump_stack(); + csd = &per_cpu(backtrace_csd, cpu); + csd->func = handle_backtrace; + smp_call_function_single_async(cpu, csd); + } } void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { - long this_cpu = get_cpu(); - - if (cpumask_test_cpu(this_cpu, mask) && !exclude_self) - dump_stack(); - - smp_call_function_many(mask, arch_dump_stack, NULL, 1); - - put_cpu(); + nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace); } int mips_get_process_fp_mode(struct task_struct *task) @@ -691,19 +707,25 @@ int mips_get_process_fp_mode(struct task_struct *task) return value; } -static void prepare_for_fp_mode_switch(void *info) +static long prepare_for_fp_mode_switch(void *unused) { - struct mm_struct *mm = info; - - if (current->mm == mm) - lose_fpu(1); + /* + * This is icky, but we use this to simply ensure that all CPUs have + * context switched, regardless of whether they were previously running + * kernel or user code. This ensures that no CPU currently has its FPU + * enabled, or is about to attempt to enable it through any path other + * than enable_restore_fp_context() which will wait appropriately for + * fp_mode_switching to be zero. + */ + return 0; } int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) { const unsigned int known_bits = PR_FP_MODE_FR | PR_FP_MODE_FRE; struct task_struct *t; - int max_users; + struct cpumask process_cpus; + int cpu; /* If nothing to change, return right away, successfully. */ if (value == mips_get_process_fp_mode(task)) @@ -736,35 +758,7 @@ int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) if (!(value & PR_FP_MODE_FR) && raw_cpu_has_fpu && cpu_has_mips_r6) return -EOPNOTSUPP; - /* Proceed with the mode switch */ - preempt_disable(); - - /* Save FP & vector context, then disable FPU & MSA */ - if (task->signal == current->signal) - lose_fpu(1); - - /* Prevent any threads from obtaining live FP context */ - atomic_set(&task->mm->context.fp_mode_switching, 1); - smp_mb__after_atomic(); - - /* - * If there are multiple online CPUs then force any which are running - * threads in this process to lose their FPU context, which they can't - * regain until fp_mode_switching is cleared later. - */ - if (num_online_cpus() > 1) { - /* No need to send an IPI for the local CPU */ - max_users = (task->mm == current->mm) ? 1 : 0; - - if (atomic_read(¤t->mm->mm_users) > max_users) - smp_call_function(prepare_for_fp_mode_switch, - (void *)current->mm, 1); - } - - /* - * There are now no threads of the process with live FP context, so it - * is safe to proceed with the FP mode switch. - */ + /* Indicate the new FP mode in each thread */ for_each_thread(task, t) { /* Update desired FP register width */ if (value & PR_FP_MODE_FR) { @@ -781,9 +775,34 @@ int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) clear_tsk_thread_flag(t, TIF_HYBRID_FPREGS); } - /* Allow threads to use FP again */ - atomic_set(&task->mm->context.fp_mode_switching, 0); - preempt_enable(); + /* + * We need to ensure that all threads in the process have switched mode + * before returning, in order to allow userland to not worry about + * races. We can do this by forcing all CPUs that any thread in the + * process may be running on to schedule something else - in this case + * prepare_for_fp_mode_switch(). + * + * We begin by generating a mask of all CPUs that any thread in the + * process may be running on. + */ + cpumask_clear(&process_cpus); + for_each_thread(task, t) + cpumask_set_cpu(task_cpu(t), &process_cpus); + + /* + * Now we schedule prepare_for_fp_mode_switch() on each of those CPUs. + * + * The CPUs may have rescheduled already since we switched mode or + * generated the cpumask, but that doesn't matter. If the task in this + * process is scheduled out then our scheduling + * prepare_for_fp_mode_switch() will simply be redundant. If it's + * scheduled in then it will already have picked up the new FP mode + * whilst doing so. + */ + get_online_cpus(); + for_each_cpu_and(cpu, &process_cpus, cpu_online_mask) + work_on_cpu(cpu, prepare_for_fp_mode_switch, NULL); + put_online_cpus(); wake_up_var(&task->mm->context.fp_mode_switching); diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 9f6c3f2aa2e2..e5ba56c01ee0 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -41,6 +41,7 @@ #include <asm/mipsmtregs.h> #include <asm/pgtable.h> #include <asm/page.h> +#include <asm/processor.h> #include <asm/syscall.h> #include <linux/uaccess.h> #include <asm/bootinfo.h> @@ -589,9 +590,226 @@ static int fpr_set(struct task_struct *target, return err; } +#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) + +/* + * Copy the DSP context to the supplied 32-bit NT_MIPS_DSP buffer. + */ +static int dsp32_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + unsigned int start, num_regs, i; + u32 dspregs[NUM_DSP_REGS + 1]; + + BUG_ON(count % sizeof(u32)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u32); + num_regs = count / sizeof(u32); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + dspregs[i] = target->thread.dsp.dspr[i]; + break; + case NUM_DSP_REGS: + dspregs[i] = target->thread.dsp.dspcontrol; + break; + } + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); +} + +/* + * Copy the supplied 32-bit NT_MIPS_DSP buffer to the DSP context. + */ +static int dsp32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned int start, num_regs, i; + u32 dspregs[NUM_DSP_REGS + 1]; + int err; + + BUG_ON(count % sizeof(u32)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u32); + num_regs = count / sizeof(u32); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + target->thread.dsp.dspr[i] = (s32)dspregs[i]; + break; + case NUM_DSP_REGS: + target->thread.dsp.dspcontrol = (s32)dspregs[i]; + break; + } + + return 0; +} + +#endif /* CONFIG_32BIT || CONFIG_MIPS32_O32 */ + +#ifdef CONFIG_64BIT + +/* + * Copy the DSP context to the supplied 64-bit NT_MIPS_DSP buffer. + */ +static int dsp64_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + unsigned int start, num_regs, i; + u64 dspregs[NUM_DSP_REGS + 1]; + + BUG_ON(count % sizeof(u64)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u64); + num_regs = count / sizeof(u64); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + dspregs[i] = target->thread.dsp.dspr[i]; + break; + case NUM_DSP_REGS: + dspregs[i] = target->thread.dsp.dspcontrol; + break; + } + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); +} + +/* + * Copy the supplied 64-bit NT_MIPS_DSP buffer to the DSP context. + */ +static int dsp64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned int start, num_regs, i; + u64 dspregs[NUM_DSP_REGS + 1]; + int err; + + BUG_ON(count % sizeof(u64)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u64); + num_regs = count / sizeof(u64); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + target->thread.dsp.dspr[i] = dspregs[i]; + break; + case NUM_DSP_REGS: + target->thread.dsp.dspcontrol = dspregs[i]; + break; + } + + return 0; +} + +#endif /* CONFIG_64BIT */ + +/* + * Determine whether the DSP context is present. + */ +static int dsp_active(struct task_struct *target, + const struct user_regset *regset) +{ + return cpu_has_dsp ? NUM_DSP_REGS + 1 : -ENODEV; +} + +/* Copy the FP mode setting to the supplied NT_MIPS_FP_MODE buffer. */ +static int fp_mode_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + int fp_mode; + + fp_mode = mips_get_process_fp_mode(target); + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &fp_mode, 0, + sizeof(fp_mode)); +} + +/* + * Copy the supplied NT_MIPS_FP_MODE buffer to the FP mode setting. + * + * We optimize for the case where `count % sizeof(int) == 0', which + * is supposed to have been guaranteed by the kernel before calling + * us, e.g. in `ptrace_regset'. We enforce that requirement, so + * that we can safely avoid preinitializing temporaries for partial + * mode writes. + */ +static int fp_mode_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int fp_mode; + int err; + + BUG_ON(count % sizeof(int)); + + if (pos + count > sizeof(fp_mode)) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fp_mode, 0, + sizeof(fp_mode)); + if (err) + return err; + + if (count > 0) + err = mips_set_process_fp_mode(target, fp_mode); + + return err; +} + enum mips_regset { REGSET_GPR, REGSET_FPR, + REGSET_DSP, + REGSET_FP_MODE, }; struct pt_regs_offset { @@ -697,6 +915,23 @@ static const struct user_regset mips_regsets[] = { .get = fpr_get, .set = fpr_set, }, + [REGSET_DSP] = { + .core_note_type = NT_MIPS_DSP, + .n = NUM_DSP_REGS + 1, + .size = sizeof(u32), + .align = sizeof(u32), + .get = dsp32_get, + .set = dsp32_set, + .active = dsp_active, + }, + [REGSET_FP_MODE] = { + .core_note_type = NT_MIPS_FP_MODE, + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .get = fp_mode_get, + .set = fp_mode_set, + }, }; static const struct user_regset_view user_mips_view = { @@ -728,6 +963,23 @@ static const struct user_regset mips64_regsets[] = { .get = fpr_get, .set = fpr_set, }, + [REGSET_DSP] = { + .core_note_type = NT_MIPS_DSP, + .n = NUM_DSP_REGS + 1, + .size = sizeof(u64), + .align = sizeof(u64), + .get = dsp64_get, + .set = dsp64_set, + .active = dsp_active, + }, + [REGSET_FP_MODE] = { + .core_note_type = NT_MIPS_FP_MODE, + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .get = fp_mode_get, + .set = fp_mode_set, + }, }; static const struct user_regset_view user_mips64_view = { @@ -856,7 +1108,7 @@ long arch_ptrace(struct task_struct *child, long request, goto out; } dregs = __get_dsp_regs(child); - tmp = (unsigned long) (dregs[addr - DSP_BASE]); + tmp = dregs[addr - DSP_BASE]; break; } case DSP_CONTROL: diff --git a/arch/mips/kernel/ptrace32.c b/arch/mips/kernel/ptrace32.c index 7edc629304c8..bc348d44d151 100644 --- a/arch/mips/kernel/ptrace32.c +++ b/arch/mips/kernel/ptrace32.c @@ -142,7 +142,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, goto out; } dregs = __get_dsp_regs(child); - tmp = (unsigned long) (dregs[addr - DSP_BASE]); + tmp = dregs[addr - DSP_BASE]; break; } case DSP_CONTROL: diff --git a/arch/mips/kernel/relocate_kernel.S b/arch/mips/kernel/relocate_kernel.S index c6bbf2165051..419c92197b2f 100644 --- a/arch/mips/kernel/relocate_kernel.S +++ b/arch/mips/kernel/relocate_kernel.S @@ -85,7 +85,7 @@ done: #ifdef CONFIG_CPU_CAVIUM_OCTEON /* We need to flush I-cache before jumping to new kernel. - * Unfortunatelly, this code is cpu-specific. + * Unfortunately, this code is cpu-specific. */ .set push .set noreorder @@ -145,7 +145,7 @@ LEAF(kexec_smp_wait) #endif /* All parameters to new kernel are passed in registers a0-a3. - * kexec_args[0..3] are uses to prepare register values. + * kexec_args[0..3] are used to prepare register values. */ kexec_args: diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 2c96c0c68116..c71d1eb7da59 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -36,6 +36,7 @@ #include <asm/cdmm.h> #include <asm/cpu.h> #include <asm/debug.h> +#include <asm/dma-coherence.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp-ops.h> @@ -84,6 +85,11 @@ static struct resource bss_resource = { .name = "Kernel bss", }; static void *detect_magic __initdata = detect_memory_region; +#ifdef CONFIG_MIPS_AUTO_PFN_OFFSET +unsigned long ARCH_PFN_OFFSET; +EXPORT_SYMBOL(ARCH_PFN_OFFSET); +#endif + void __init add_memory_region(phys_addr_t start, phys_addr_t size, long type) { int x = boot_mem_map.nr_map; @@ -441,6 +447,12 @@ static void __init bootmem_init(void) mapstart = max(reserved_end, start); } + if (min_low_pfn >= max_low_pfn) + panic("Incorrect memory mapping !!!"); + +#ifdef CONFIG_MIPS_AUTO_PFN_OFFSET + ARCH_PFN_OFFSET = PFN_UP(ramstart); +#else /* * Reserve any memory between the start of RAM and PHYS_OFFSET */ @@ -448,8 +460,6 @@ static void __init bootmem_init(void) add_memory_region(PHYS_OFFSET, ramstart - PHYS_OFFSET, BOOT_MEM_RESERVED); - if (min_low_pfn >= max_low_pfn) - panic("Incorrect memory mapping !!!"); if (min_low_pfn > ARCH_PFN_OFFSET) { pr_info("Wasting %lu bytes for tracking %lu unused pages\n", (min_low_pfn - ARCH_PFN_OFFSET) * sizeof(struct page), @@ -459,6 +469,7 @@ static void __init bootmem_init(void) ARCH_PFN_OFFSET - min_low_pfn); } min_low_pfn = ARCH_PFN_OFFSET; +#endif /* * Determine low and high memory ranges @@ -1055,3 +1066,26 @@ static int __init debugfs_mips(void) } arch_initcall(debugfs_mips); #endif + +#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT) +/* User defined DMA coherency from command line. */ +enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; +EXPORT_SYMBOL_GPL(coherentio); +int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ + +static int __init setcoherentio(char *str) +{ + coherentio = IO_COHERENCE_ENABLED; + pr_info("Hardware DMA cache coherency (command line)\n"); + return 0; +} +early_param("coherentio", setcoherentio); + +static int __init setnocoherentio(char *str) +{ + coherentio = IO_COHERENCE_DISABLED; + pr_info("Software DMA cache coherency (command line)\n"); + return 0; +} +early_param("nocoherentio", setnocoherentio); +#endif diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index 0a9cfe7a0372..109ed163a6a6 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -592,13 +592,15 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, #endif #ifdef CONFIG_TRAD_SIGNALS -asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys_sigreturn(void) { struct sigframe __user *frame; + struct pt_regs *regs; sigset_t blocked; int sig; - frame = (struct sigframe __user *) regs.regs[29]; + regs = current_pt_regs(); + frame = (struct sigframe __user *)regs->regs[29]; if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&blocked, &frame->sf_mask, sizeof(blocked))) @@ -606,7 +608,7 @@ asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) set_current_blocked(&blocked); - sig = restore_sigcontext(®s, &frame->sf_sc); + sig = restore_sigcontext(regs, &frame->sf_sc); if (sig < 0) goto badframe; else if (sig) @@ -618,8 +620,8 @@ asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: @@ -627,13 +629,15 @@ badframe: } #endif /* CONFIG_TRAD_SIGNALS */ -asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys_rt_sigreturn(void) { struct rt_sigframe __user *frame; + struct pt_regs *regs; sigset_t set; int sig; - frame = (struct rt_sigframe __user *) regs.regs[29]; + regs = current_pt_regs(); + frame = (struct rt_sigframe __user *)regs->regs[29]; if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->rs_uc.uc_sigmask, sizeof(set))) @@ -641,7 +645,7 @@ asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) set_current_blocked(&set); - sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); + sig = restore_sigcontext(regs, &frame->rs_uc.uc_mcontext); if (sig < 0) goto badframe; else if (sig) @@ -656,8 +660,8 @@ asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c index b672cebb4a1a..8f65aaf9206d 100644 --- a/arch/mips/kernel/signal_n32.c +++ b/arch/mips/kernel/signal_n32.c @@ -64,13 +64,15 @@ struct rt_sigframe_n32 { struct ucontextn32 rs_uc; }; -asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sysn32_rt_sigreturn(void) { struct rt_sigframe_n32 __user *frame; + struct pt_regs *regs; sigset_t set; int sig; - frame = (struct rt_sigframe_n32 __user *) regs.regs[29]; + regs = current_pt_regs(); + frame = (struct rt_sigframe_n32 __user *)regs->regs[29]; if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) @@ -78,7 +80,7 @@ asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) set_current_blocked(&set); - sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); + sig = restore_sigcontext(regs, &frame->rs_uc.uc_mcontext); if (sig < 0) goto badframe; else if (sig) @@ -93,8 +95,8 @@ asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: diff --git a/arch/mips/kernel/signal_o32.c b/arch/mips/kernel/signal_o32.c index 2b3572fb5f1b..b6e3ddef48a0 100644 --- a/arch/mips/kernel/signal_o32.c +++ b/arch/mips/kernel/signal_o32.c @@ -151,13 +151,15 @@ static int setup_frame_32(void *sig_return, struct ksignal *ksig, return 0; } -asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys32_rt_sigreturn(void) { struct rt_sigframe32 __user *frame; + struct pt_regs *regs; sigset_t set; int sig; - frame = (struct rt_sigframe32 __user *) regs.regs[29]; + regs = current_pt_regs(); + frame = (struct rt_sigframe32 __user *)regs->regs[29]; if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) @@ -165,7 +167,7 @@ asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) set_current_blocked(&set); - sig = restore_sigcontext32(®s, &frame->rs_uc.uc_mcontext); + sig = restore_sigcontext32(regs, &frame->rs_uc.uc_mcontext); if (sig < 0) goto badframe; else if (sig) @@ -180,8 +182,8 @@ asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: @@ -251,13 +253,15 @@ struct mips_abi mips_abi_32 = { }; -asmlinkage void sys32_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys32_sigreturn(void) { struct sigframe32 __user *frame; + struct pt_regs *regs; sigset_t blocked; int sig; - frame = (struct sigframe32 __user *) regs.regs[29]; + regs = current_pt_regs(); + frame = (struct sigframe32 __user *)regs->regs[29]; if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&blocked, &frame->sf_mask)) @@ -265,7 +269,7 @@ asmlinkage void sys32_sigreturn(nabi_no_regargs struct pt_regs regs) set_current_blocked(&blocked); - sig = restore_sigcontext32(®s, &frame->sf_sc); + sig = restore_sigcontext32(regs, &frame->sf_sc); if (sig < 0) goto badframe; else if (sig) @@ -277,8 +281,8 @@ asmlinkage void sys32_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index d67fa74622ee..f8871d5b7eb3 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -351,6 +351,7 @@ static void __show_regs(const struct pt_regs *regs) void show_regs(struct pt_regs *regs) { __show_regs((struct pt_regs *)regs); + dump_stack(); } void show_registers(struct pt_regs *regs) @@ -1220,13 +1221,6 @@ static int enable_restore_fp_context(int msa) { int err, was_fpu_owner, prior_msa; - /* - * If an FP mode switch is currently underway, wait for it to - * complete before proceeding. - */ - wait_var_event(¤t->mm->context.fp_mode_switching, - !atomic_read(¤t->mm->context.fp_mode_switching)); - if (!used_math()) { /* First time FP context user. */ preempt_disable(); diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 7cd76f93a438..f7ea8e21656b 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -515,7 +515,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, dvcpu->arch.wait = 0; if (swq_has_sleeper(&dvcpu->wq)) - swake_up(&dvcpu->wq); + swake_up_one(&dvcpu->wq); return 0; } @@ -1204,7 +1204,7 @@ static void kvm_mips_comparecount_func(unsigned long data) vcpu->arch.wait = 0; if (swq_has_sleeper(&vcpu->wq)) - swake_up(&vcpu->wq); + swake_up_one(&vcpu->wq); } /* low level hrtimer wake routine */ diff --git a/arch/mips/lantiq/early_printk.c b/arch/mips/lantiq/early_printk.c index 44bccaee822b..c4aa140b7c91 100644 --- a/arch/mips/lantiq/early_printk.c +++ b/arch/mips/lantiq/early_printk.c @@ -8,6 +8,7 @@ #include <linux/cpu.h> #include <lantiq_soc.h> +#include <asm/setup.h> #define ASC_BUF 1024 #define LTQ_ASC_FSTAT ((u32 *)(LTQ_EARLY_ASC + 0x0048)) diff --git a/arch/mips/lantiq/prom.c b/arch/mips/lantiq/prom.c index 9ff7ccde9de0..d984bd5c2ec5 100644 --- a/arch/mips/lantiq/prom.c +++ b/arch/mips/lantiq/prom.c @@ -9,7 +9,6 @@ #include <linux/export.h> #include <linux/clk.h> #include <linux/bootmem.h> -#include <linux/of_platform.h> #include <linux/of_fdt.h> #include <asm/bootinfo.h> @@ -114,10 +113,3 @@ void __init prom_init(void) panic("failed to register_vsmp_smp_ops()"); #endif } - -int __init plat_of_setup(void) -{ - return of_platform_default_populate(NULL, NULL, NULL); -} - -arch_initcall(plat_of_setup); diff --git a/arch/mips/lantiq/xway/dma.c b/arch/mips/lantiq/xway/dma.c index 805b3a6ab2d6..4b9fbb6744ad 100644 --- a/arch/mips/lantiq/xway/dma.c +++ b/arch/mips/lantiq/xway/dma.c @@ -130,10 +130,9 @@ ltq_dma_alloc(struct ltq_dma_channel *ch) unsigned long flags; ch->desc = 0; - ch->desc_base = dma_alloc_coherent(NULL, + ch->desc_base = dma_zalloc_coherent(NULL, LTQ_DESC_NUM * LTQ_DESC_SIZE, &ch->phys, GFP_ATOMIC); - memset(ch->desc_base, 0, LTQ_DESC_NUM * LTQ_DESC_SIZE); spin_lock_irqsave(<q_dma_lock, flags); ltq_dma_w32(ch->nr, LTQ_DMA_CS); diff --git a/arch/mips/lasat/prom.c b/arch/mips/lasat/prom.c index 17e15b50a551..37b8fc5b9ac9 100644 --- a/arch/mips/lasat/prom.c +++ b/arch/mips/lasat/prom.c @@ -13,6 +13,7 @@ #include <asm/bootinfo.h> #include <asm/lasat/lasat.h> #include <asm/cpu.h> +#include <asm/setup.h> #include "at93c.h" #include <asm/lasat/eeprom.h> diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S index 1cc306520a55..3a6f34ef5ffc 100644 --- a/arch/mips/lib/memset.S +++ b/arch/mips/lib/memset.S @@ -195,6 +195,7 @@ #endif #else PTR_SUBU t0, $0, a2 + move a2, zero /* No remaining longs */ PTR_ADDIU t0, 1 STORE_BYTE(0) STORE_BYTE(1) @@ -231,16 +232,25 @@ #ifdef CONFIG_CPU_MIPSR6 .Lbyte_fixup\@: - PTR_SUBU a2, $0, t0 + /* + * unset_bytes = (#bytes - (#unaligned bytes)) - (-#unaligned bytes remaining + 1) + 1 + * a2 = a2 - t0 + 1 + */ + PTR_SUBU a2, t0 jr ra PTR_ADDIU a2, 1 #endif /* CONFIG_CPU_MIPSR6 */ .Lfirst_fixup\@: + /* unset_bytes already in a2 */ jr ra nop .Lfwd_fixup\@: + /* + * unset_bytes = partial_start_addr + #bytes - fault_addr + * a2 = t1 + (a2 & 3f) - $28->task->BUADDR + */ PTR_L t0, TI_TASK($28) andi a2, 0x3f LONG_L t0, THREAD_BUADDR(t0) @@ -249,6 +259,10 @@ LONG_SUBU a2, t0 .Lpartial_fixup\@: + /* + * unset_bytes = partial_end_addr + #bytes - fault_addr + * a2 = a0 + (a2 & STORMASK) - $28->task->BUADDR + */ PTR_L t0, TI_TASK($28) andi a2, STORMASK LONG_L t0, THREAD_BUADDR(t0) @@ -257,10 +271,15 @@ LONG_SUBU a2, t0 .Llast_fixup\@: + /* unset_bytes already in a2 */ jr ra nop .Lsmall_fixup\@: + /* + * unset_bytes = end_addr - current_addr + 1 + * a2 = t1 - a0 + 1 + */ PTR_SUBU a2, t1, a0 jr ra PTR_ADDIU a2, 1 diff --git a/arch/mips/loongson32/Platform b/arch/mips/loongson32/Platform index ffe01c6d0037..a0dbb3b2f2de 100644 --- a/arch/mips/loongson32/Platform +++ b/arch/mips/loongson32/Platform @@ -1,8 +1,4 @@ -cflags-$(CONFIG_CPU_LOONGSON1) += \ - $(call cc-option,-march=mips32r2,-mips32r2 -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS32) \ - -Wa,-mips32r2 -Wa,--trap - +cflags-$(CONFIG_CPU_LOONGSON1) += -march=mips32 -Wa,--trap platform-$(CONFIG_MACH_LOONGSON32) += loongson32/ cflags-$(CONFIG_MACH_LOONGSON32) += -I$(srctree)/arch/mips/include/asm/mach-loongson32 -load-$(CONFIG_LOONGSON1_LS1B) += 0xffffffff80100000 -load-$(CONFIG_LOONGSON1_LS1C) += 0xffffffff80100000 +load-$(CONFIG_CPU_LOONGSON1) += 0xffffffff80100000 diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig index c79e6a565572..c865b4b9b775 100644 --- a/arch/mips/loongson64/Kconfig +++ b/arch/mips/loongson64/Kconfig @@ -91,7 +91,6 @@ config LOONGSON_MACH3X select LOONGSON_MC146818 select ZONE_DMA32 select LEFI_FIRMWARE_INTERFACE - select PHYS48_TO_HT40 help Generic Loongson 3 family machines utilize the 3A/3B revision of Loongson processor and RS780/SBX00 chipset. @@ -130,10 +129,6 @@ config LOONGSON_UART_BASE default y depends on EARLY_PRINTK || SERIAL_8250 -config PHYS48_TO_HT40 - bool - default y if CPU_LOONGSON3 - config LOONGSON_MC146818 bool default n diff --git a/arch/mips/loongson64/common/Makefile b/arch/mips/loongson64/common/Makefile index 8235ac7eac95..57ee03022941 100644 --- a/arch/mips/loongson64/common/Makefile +++ b/arch/mips/loongson64/common/Makefile @@ -6,6 +6,7 @@ obj-y += setup.o init.o cmdline.o env.o time.o reset.o irq.o \ bonito-irq.o mem.o machtype.o platform.o serial.o obj-$(CONFIG_PCI) += pci.o +obj-$(CONFIG_CPU_LOONGSON2) += dma.o # # Serial port support @@ -25,8 +26,3 @@ obj-$(CONFIG_CS5536) += cs5536/ # obj-$(CONFIG_SUSPEND) += pm.o - -# -# Big Memory (SWIOTLB) Support -# -obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o diff --git a/arch/mips/loongson64/common/cs5536/cs5536_ohci.c b/arch/mips/loongson64/common/cs5536/cs5536_ohci.c index f7c905e50dc4..92dc6bafc127 100644 --- a/arch/mips/loongson64/common/cs5536/cs5536_ohci.c +++ b/arch/mips/loongson64/common/cs5536/cs5536_ohci.c @@ -138,7 +138,7 @@ u32 pci_ohci_read_reg(int reg) break; case PCI_OHCI_INT_REG: _rdmsr(DIVIL_MSR_REG(PIC_YSEL_LOW), &hi, &lo); - if ((lo & 0x00000f00) == CS5536_USB_INTR) + if (((lo >> PIC_YSEL_LOW_USB_SHIFT) & 0xf) == CS5536_USB_INTR) conf_data = 1; break; default: diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c deleted file mode 100644 index 6a739f8ae110..000000000000 --- a/arch/mips/loongson64/common/dma-swiotlb.c +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/mm.h> -#include <linux/init.h> -#include <linux/dma-mapping.h> -#include <linux/scatterlist.h> -#include <linux/swiotlb.h> -#include <linux/bootmem.h> - -#include <asm/bootinfo.h> -#include <boot_param.h> -#include <dma-coherence.h> - -static void *loongson_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) -{ - void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs); - - mb(); - return ret; -} - -static dma_addr_t loongson_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size, - dir, attrs); - mb(); - return daddr; -} - -static int loongson_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - int r = swiotlb_map_sg_attrs(dev, sg, nents, dir, attrs); - mb(); - - return r; -} - -static void loongson_dma_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, - enum dma_data_direction dir) -{ - swiotlb_sync_single_for_device(dev, dma_handle, size, dir); - mb(); -} - -static void loongson_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sg, int nents, - enum dma_data_direction dir) -{ - swiotlb_sync_sg_for_device(dev, sg, nents, dir); - mb(); -} - -static int loongson_dma_supported(struct device *dev, u64 mask) -{ - if (mask > DMA_BIT_MASK(loongson_sysconf.dma_mask_bits)) - return 0; - return swiotlb_dma_supported(dev, mask); -} - -dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - long nid; -#ifdef CONFIG_PHYS48_TO_HT40 - /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from - * Loongson-3's 48bit address space and embed it into 40bit */ - nid = (paddr >> 44) & 0x3; - paddr = ((nid << 44) ^ paddr) | (nid << 37); -#endif - return paddr; -} - -phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) -{ - long nid; -#ifdef CONFIG_PHYS48_TO_HT40 - /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from - * Loongson-3's 48bit address space and embed it into 40bit */ - nid = (daddr >> 37) & 0x3; - daddr = ((nid << 37) ^ daddr) | (nid << 44); -#endif - return daddr; -} - -static const struct dma_map_ops loongson_dma_map_ops = { - .alloc = loongson_dma_alloc_coherent, - .free = swiotlb_free, - .map_page = loongson_dma_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = loongson_dma_map_sg, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = loongson_dma_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = loongson_dma_sync_sg_for_device, - .mapping_error = swiotlb_dma_mapping_error, - .dma_supported = loongson_dma_supported, -}; - -void __init plat_swiotlb_setup(void) -{ - swiotlb_init(1); - mips_dma_map_ops = &loongson_dma_map_ops; -} diff --git a/arch/mips/loongson64/common/dma.c b/arch/mips/loongson64/common/dma.c new file mode 100644 index 000000000000..48f04126bde2 --- /dev/null +++ b/arch/mips/loongson64/common/dma.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/dma-direct.h> + +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + return paddr | 0x80000000; +} + +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) +{ +#if defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT) + if (dma_addr > 0x8fffffff) + return dma_addr; + return dma_addr & 0x0fffffff; +#else + return dma_addr & 0x7fffffff; +#endif +} diff --git a/arch/mips/loongson64/common/early_printk.c b/arch/mips/loongson64/common/early_printk.c index 6ca632e529dc..a782e2b24747 100644 --- a/arch/mips/loongson64/common/early_printk.c +++ b/arch/mips/loongson64/common/early_printk.c @@ -10,6 +10,7 @@ * option) any later version. */ #include <linux/serial_reg.h> +#include <asm/setup.h> #include <loongson.h> diff --git a/arch/mips/loongson64/common/env.c b/arch/mips/loongson64/common/env.c index 1e8a955ae5a8..8f68ee02a8c2 100644 --- a/arch/mips/loongson64/common/env.c +++ b/arch/mips/loongson64/common/env.c @@ -198,7 +198,8 @@ void __init prom_init_env(void) break; case PRID_REV_LOONGSON3A_R1: case PRID_REV_LOONGSON3A_R2: - case PRID_REV_LOONGSON3A_R3: + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: cpu_clock_freq = 900000000; break; case PRID_REV_LOONGSON3B_R1: diff --git a/arch/mips/loongson64/loongson-3/Makefile b/arch/mips/loongson64/loongson-3/Makefile index 44bc1482158b..b5a0c2fa5446 100644 --- a/arch/mips/loongson64/loongson-3/Makefile +++ b/arch/mips/loongson64/loongson-3/Makefile @@ -1,7 +1,7 @@ # # Makefile for Loongson-3 family machines # -obj-y += irq.o cop2-ex.o platform.o acpi_init.o +obj-y += irq.o cop2-ex.o platform.o acpi_init.o dma.o obj-$(CONFIG_SMP) += smp.o diff --git a/arch/mips/loongson64/loongson-3/dma.c b/arch/mips/loongson64/loongson-3/dma.c new file mode 100644 index 000000000000..5e86635f71db --- /dev/null +++ b/arch/mips/loongson64/loongson-3/dma.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/dma-direct.h> +#include <linux/init.h> +#include <linux/swiotlb.h> + +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from + * Loongson-3's 48bit address space and embed it into 40bit */ + long nid = (paddr >> 44) & 0x3; + return ((nid << 44) ^ paddr) | (nid << 37); +} + +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) +{ + /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from + * Loongson-3's 48bit address space and embed it into 40bit */ + long nid = (daddr >> 37) & 0x3; + return ((nid << 37) ^ daddr) | (nid << 44); +} + +void __init plat_swiotlb_setup(void) +{ + swiotlb_init(1); +} diff --git a/arch/mips/loongson64/loongson-3/smp.c b/arch/mips/loongson64/loongson-3/smp.c index 8501109bb0f0..fea95d003269 100644 --- a/arch/mips/loongson64/loongson-3/smp.c +++ b/arch/mips/loongson64/loongson-3/smp.c @@ -682,7 +682,8 @@ void play_dead(void) (void *)CKSEG1ADDR((unsigned long)loongson3a_r1_play_dead); break; case PRID_REV_LOONGSON3A_R2: - case PRID_REV_LOONGSON3A_R3: + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: play_dead_at_ckseg1 = (void *)CKSEG1ADDR((unsigned long)loongson3a_r2r3_play_dead); break; diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index c463bdad45c7..3e5bb203c95a 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -3,7 +3,7 @@ # Makefile for the Linux/MIPS-specific parts of the memory manager. # -obj-y += cache.o dma-default.o extable.o fault.o \ +obj-y += cache.o extable.o fault.o \ gup.o init.o mmap.o page.o page-funcs.o \ pgtable.o tlbex.o tlbex-fault.o tlb-funcs.o @@ -17,6 +17,7 @@ obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o obj-$(CONFIG_64BIT) += pgtable-64.o obj-$(CONFIG_HIGHMEM) += highmem.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_DMA_NONCOHERENT) += dma-noncoherent.o obj-$(CONFIG_CPU_R4K_CACHE_TLB) += c-r4k.o cex-gen.o tlb-r4k.o obj-$(CONFIG_CPU_R3000) += c-r3k.o tlb-r3k.o diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index e12dfa48b478..a9ef057c79fe 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -830,12 +830,13 @@ static void r4k_flush_icache_user_range(unsigned long start, unsigned long end) return __r4k_flush_icache_range(start, end, true); } -#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) +#ifdef CONFIG_DMA_NONCOHERENT static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) { /* Catch bad driver code */ - BUG_ON(size == 0); + if (WARN_ON(size == 0)) + return; preempt_disable(); if (cpu_has_inclusive_pcaches) { @@ -871,7 +872,8 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) { /* Catch bad driver code */ - BUG_ON(size == 0); + if (WARN_ON(size == 0)) + return; preempt_disable(); if (cpu_has_inclusive_pcaches) { @@ -904,7 +906,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) bc_inv(addr, size); __sync(); } -#endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */ +#endif /* CONFIG_DMA_NONCOHERENT */ struct flush_cache_sigtramp_args { struct mm_struct *mm; @@ -1505,6 +1507,14 @@ static void probe_pcache(void) if (c->dcache.flags & MIPS_CACHE_PINDEX) c->dcache.flags &= ~MIPS_CACHE_ALIASES; + /* + * In systems with CM the icache fills from L2 or closer caches, and + * thus sees remote stores without needing to write them back any + * further than that. + */ + if (mips_cm_present()) + c->icache.flags |= MIPS_IC_SNOOPS_REMOTE; + switch (current_cpu_type()) { case CPU_20KC: /* diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 0d3c656feba0..70a523151ff3 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -56,7 +56,7 @@ EXPORT_SYMBOL_GPL(local_flush_data_cache_page); EXPORT_SYMBOL(flush_data_cache_page); EXPORT_SYMBOL(flush_icache_all); -#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) +#ifdef CONFIG_DMA_NONCOHERENT /* DMA cache operations. */ void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size); @@ -65,7 +65,7 @@ void (*_dma_cache_inv)(unsigned long start, unsigned long size); EXPORT_SYMBOL(_dma_cache_wback_inv); -#endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */ +#endif /* CONFIG_DMA_NONCOHERENT */ /* * We could optimize the case where the cache argument is not BCACHE but diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c deleted file mode 100644 index f9fef0028ca2..000000000000 --- a/arch/mips/mm/dma-default.c +++ /dev/null @@ -1,404 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> - * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> - * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. - */ - -#include <linux/types.h> -#include <linux/dma-mapping.h> -#include <linux/mm.h> -#include <linux/export.h> -#include <linux/scatterlist.h> -#include <linux/string.h> -#include <linux/gfp.h> -#include <linux/highmem.h> -#include <linux/dma-contiguous.h> - -#include <asm/cache.h> -#include <asm/cpu-type.h> -#include <asm/io.h> - -#include <dma-coherence.h> - -#if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT) -/* User defined DMA coherency from command line. */ -enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; -EXPORT_SYMBOL_GPL(coherentio); -int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ - -static int __init setcoherentio(char *str) -{ - coherentio = IO_COHERENCE_ENABLED; - pr_info("Hardware DMA cache coherency (command line)\n"); - return 0; -} -early_param("coherentio", setcoherentio); - -static int __init setnocoherentio(char *str) -{ - coherentio = IO_COHERENCE_DISABLED; - pr_info("Software DMA cache coherency (command line)\n"); - return 0; -} -early_param("nocoherentio", setnocoherentio); -#endif - -static inline struct page *dma_addr_to_page(struct device *dev, - dma_addr_t dma_addr) -{ - return pfn_to_page( - plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT); -} - -/* - * The affected CPUs below in 'cpu_needs_post_dma_flush()' can - * speculatively fill random cachelines with stale data at any time, - * requiring an extra flush post-DMA. - * - * Warning on the terminology - Linux calls an uncached area coherent; - * MIPS terminology calls memory areas with hardware maintained coherency - * coherent. - * - * Note that the R14000 and R16000 should also be checked for in this - * condition. However this function is only called on non-I/O-coherent - * systems and only the R10000 and R12000 are used in such systems, the - * SGI IP28 Indigo² rsp. SGI IP32 aka O2. - */ -static inline bool cpu_needs_post_dma_flush(struct device *dev) -{ - if (plat_device_is_coherent(dev)) - return false; - - switch (boot_cpu_type()) { - case CPU_R10000: - case CPU_R12000: - case CPU_BMIPS5000: - return true; - - default: - /* - * Presence of MAARs suggests that the CPU supports - * speculatively prefetching data, and therefore requires - * the post-DMA flush/invalidate. - */ - return cpu_has_maar; - } -} - -static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp) -{ - gfp_t dma_flag; - -#ifdef CONFIG_ISA - if (dev == NULL) - dma_flag = __GFP_DMA; - else -#endif -#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA) - if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(32)) - dma_flag = __GFP_DMA; - else if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) - dma_flag = __GFP_DMA32; - else -#endif -#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA) - if (dev == NULL || dev->coherent_dma_mask < DMA_BIT_MASK(64)) - dma_flag = __GFP_DMA32; - else -#endif -#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32) - if (dev == NULL || - dev->coherent_dma_mask < DMA_BIT_MASK(sizeof(phys_addr_t) * 8)) - dma_flag = __GFP_DMA; - else -#endif - dma_flag = 0; - - /* Don't invoke OOM killer */ - gfp |= __GFP_NORETRY; - - return gfp | dma_flag; -} - -static void *mips_dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) -{ - void *ret; - struct page *page = NULL; - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - - gfp = massage_gfp_flags(dev, gfp); - - if (IS_ENABLED(CONFIG_DMA_CMA) && gfpflags_allow_blocking(gfp)) - page = dma_alloc_from_contiguous(dev, count, get_order(size), - gfp); - if (!page) - page = alloc_pages(gfp, get_order(size)); - - if (!page) - return NULL; - - ret = page_address(page); - memset(ret, 0, size); - *dma_handle = plat_map_dma_mem(dev, ret, size); - if (!(attrs & DMA_ATTR_NON_CONSISTENT) && - !plat_device_is_coherent(dev)) { - dma_cache_wback_inv((unsigned long) ret, size); - ret = UNCAC_ADDR(ret); - } - - return ret; -} - -static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) -{ - unsigned long addr = (unsigned long) vaddr; - unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - struct page *page = NULL; - - plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); - - if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !plat_device_is_coherent(dev)) - addr = CAC_ADDR(addr); - - page = virt_to_page((void *) addr); - - if (!dma_release_from_contiguous(dev, page, count)) - __free_pages(page, get_order(size)); -} - -static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - unsigned long user_count = vma_pages(vma); - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long addr = (unsigned long)cpu_addr; - unsigned long off = vma->vm_pgoff; - unsigned long pfn; - int ret = -ENXIO; - - if (!plat_device_is_coherent(dev)) - addr = CAC_ADDR(addr); - - pfn = page_to_pfn(virt_to_page((void *)addr)); - - if (attrs & DMA_ATTR_WRITE_COMBINE) - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - else - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < count && user_count <= (count - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - user_count << PAGE_SHIFT, - vma->vm_page_prot); - } - - return ret; -} - -static inline void __dma_sync_virtual(void *addr, size_t size, - enum dma_data_direction direction) -{ - switch (direction) { - case DMA_TO_DEVICE: - dma_cache_wback((unsigned long)addr, size); - break; - - case DMA_FROM_DEVICE: - dma_cache_inv((unsigned long)addr, size); - break; - - case DMA_BIDIRECTIONAL: - dma_cache_wback_inv((unsigned long)addr, size); - break; - - default: - BUG(); - } -} - -/* - * A single sg entry may refer to multiple physically contiguous - * pages. But we still need to process highmem pages individually. - * If highmem is not configured then the bulk of this loop gets - * optimized out. - */ -static inline void __dma_sync(struct page *page, - unsigned long offset, size_t size, enum dma_data_direction direction) -{ - size_t left = size; - - do { - size_t len = left; - - if (PageHighMem(page)) { - void *addr; - - if (offset + len > PAGE_SIZE) { - if (offset >= PAGE_SIZE) { - page += offset >> PAGE_SHIFT; - offset &= ~PAGE_MASK; - } - len = PAGE_SIZE - offset; - } - - addr = kmap_atomic(page); - __dma_sync_virtual(addr + offset, len, direction); - kunmap_atomic(addr); - } else - __dma_sync_virtual(page_address(page) + offset, - size, direction); - offset = 0; - page++; - left -= len; - } while (left); -} - -static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction direction, unsigned long attrs) -{ - if (cpu_needs_post_dma_flush(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_sync(dma_addr_to_page(dev, dma_addr), - dma_addr & ~PAGE_MASK, size, direction); - plat_post_dma_flush(dev); - plat_unmap_dma_mem(dev, dma_addr, size, direction); -} - -static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction direction, unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sglist, sg, nents, i) { - if (!plat_device_is_coherent(dev) && - !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_sync(sg_page(sg), sg->offset, sg->length, - direction); -#ifdef CONFIG_NEED_SG_DMA_LENGTH - sg->dma_length = sg->length; -#endif - sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) + - sg->offset; - } - - return nents; -} - -static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction direction, - unsigned long attrs) -{ - if (!plat_device_is_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_sync(page, offset, size, direction); - - return plat_map_dma_mem_page(dev, page) + offset; -} - -static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nhwentries, enum dma_data_direction direction, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sglist, sg, nhwentries, i) { - if (!plat_device_is_coherent(dev) && - !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - direction != DMA_TO_DEVICE) - __dma_sync(sg_page(sg), sg->offset, sg->length, - direction); - plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction); - } -} - -static void mips_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) -{ - if (cpu_needs_post_dma_flush(dev)) - __dma_sync(dma_addr_to_page(dev, dma_handle), - dma_handle & ~PAGE_MASK, size, direction); - plat_post_dma_flush(dev); -} - -static void mips_dma_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) -{ - if (!plat_device_is_coherent(dev)) - __dma_sync(dma_addr_to_page(dev, dma_handle), - dma_handle & ~PAGE_MASK, size, direction); -} - -static void mips_dma_sync_sg_for_cpu(struct device *dev, - struct scatterlist *sglist, int nelems, - enum dma_data_direction direction) -{ - int i; - struct scatterlist *sg; - - if (cpu_needs_post_dma_flush(dev)) { - for_each_sg(sglist, sg, nelems, i) { - __dma_sync(sg_page(sg), sg->offset, sg->length, - direction); - } - } - plat_post_dma_flush(dev); -} - -static void mips_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sglist, int nelems, - enum dma_data_direction direction) -{ - int i; - struct scatterlist *sg; - - if (!plat_device_is_coherent(dev)) { - for_each_sg(sglist, sg, nelems, i) { - __dma_sync(sg_page(sg), sg->offset, sg->length, - direction); - } - } -} - -static int mips_dma_supported(struct device *dev, u64 mask) -{ - return plat_dma_supported(dev, mask); -} - -static void mips_dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); - - if (!plat_device_is_coherent(dev)) - __dma_sync_virtual(vaddr, size, direction); -} - -static const struct dma_map_ops mips_default_dma_map_ops = { - .alloc = mips_dma_alloc_coherent, - .free = mips_dma_free_coherent, - .mmap = mips_dma_mmap, - .map_page = mips_dma_map_page, - .unmap_page = mips_dma_unmap_page, - .map_sg = mips_dma_map_sg, - .unmap_sg = mips_dma_unmap_sg, - .sync_single_for_cpu = mips_dma_sync_single_for_cpu, - .sync_single_for_device = mips_dma_sync_single_for_device, - .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu, - .sync_sg_for_device = mips_dma_sync_sg_for_device, - .dma_supported = mips_dma_supported, - .cache_sync = mips_dma_cache_sync, -}; - -const struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops; -EXPORT_SYMBOL(mips_dma_map_ops); diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c new file mode 100644 index 000000000000..2aca1236af36 --- /dev/null +++ b/arch/mips/mm/dma-noncoherent.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> + * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> + * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. + */ +#include <linux/dma-direct.h> +#include <linux/dma-noncoherent.h> +#include <linux/dma-contiguous.h> +#include <linux/highmem.h> + +#include <asm/cache.h> +#include <asm/cpu-type.h> +#include <asm/dma-coherence.h> +#include <asm/io.h> + +#ifdef CONFIG_DMA_PERDEV_COHERENT +static inline int dev_is_coherent(struct device *dev) +{ + return dev->archdata.dma_coherent; +} +#else +static inline int dev_is_coherent(struct device *dev) +{ + switch (coherentio) { + default: + case IO_COHERENCE_DEFAULT: + return hw_coherentio; + case IO_COHERENCE_ENABLED: + return 1; + case IO_COHERENCE_DISABLED: + return 0; + } +} +#endif /* CONFIG_DMA_PERDEV_COHERENT */ + +/* + * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively + * fill random cachelines with stale data at any time, requiring an extra + * flush post-DMA. + * + * Warning on the terminology - Linux calls an uncached area coherent; MIPS + * terminology calls memory areas with hardware maintained coherency coherent. + * + * Note that the R14000 and R16000 should also be checked for in this condition. + * However this function is only called on non-I/O-coherent systems and only the + * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp. + * SGI IP32 aka O2. + */ +static inline bool cpu_needs_post_dma_flush(struct device *dev) +{ + if (dev_is_coherent(dev)) + return false; + + switch (boot_cpu_type()) { + case CPU_R10000: + case CPU_R12000: + case CPU_BMIPS5000: + return true; + default: + /* + * Presence of MAARs suggests that the CPU supports + * speculatively prefetching data, and therefore requires + * the post-DMA flush/invalidate. + */ + return cpu_has_maar; + } +} + +void *arch_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + void *ret; + + ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!ret) + return NULL; + + if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + dma_cache_wback_inv((unsigned long) ret, size); + ret = (void *)UNCAC_ADDR(ret); + } + + return ret; +} + +void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs) +{ + if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev)) + cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr); + dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); +} + +int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long user_count = vma_pages(vma); + unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long addr = (unsigned long)cpu_addr; + unsigned long off = vma->vm_pgoff; + unsigned long pfn; + int ret = -ENXIO; + + if (!dev_is_coherent(dev)) + addr = CAC_ADDR(addr); + + pfn = page_to_pfn(virt_to_page((void *)addr)); + + if (attrs & DMA_ATTR_WRITE_COMBINE) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off < count && user_count <= (count - off)) { + ret = remap_pfn_range(vma, vma->vm_start, + pfn + off, + user_count << PAGE_SHIFT, + vma->vm_page_prot); + } + + return ret; +} + +static inline void dma_sync_virt(void *addr, size_t size, + enum dma_data_direction dir) +{ + switch (dir) { + case DMA_TO_DEVICE: + dma_cache_wback((unsigned long)addr, size); + break; + + case DMA_FROM_DEVICE: + dma_cache_inv((unsigned long)addr, size); + break; + + case DMA_BIDIRECTIONAL: + dma_cache_wback_inv((unsigned long)addr, size); + break; + + default: + BUG(); + } +} + +/* + * A single sg entry may refer to multiple physically contiguous pages. But + * we still need to process highmem pages individually. If highmem is not + * configured then the bulk of this loop gets optimized out. + */ +static inline void dma_sync_phys(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) +{ + struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); + unsigned long offset = paddr & ~PAGE_MASK; + size_t left = size; + + do { + size_t len = left; + + if (PageHighMem(page)) { + void *addr; + + if (offset + len > PAGE_SIZE) { + if (offset >= PAGE_SIZE) { + page += offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + } + len = PAGE_SIZE - offset; + } + + addr = kmap_atomic(page); + dma_sync_virt(addr + offset, len, dir); + kunmap_atomic(addr); + } else + dma_sync_virt(page_address(page) + offset, size, dir); + offset = 0; + page++; + left -= len; + } while (left); +} + +void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) +{ + if (!dev_is_coherent(dev)) + dma_sync_phys(paddr, size, dir); +} + +void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) +{ + if (cpu_needs_post_dma_flush(dev)) + dma_sync_phys(paddr, size, dir); +} + +void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction direction) +{ + BUG_ON(direction == DMA_NONE); + + if (!dev_is_coherent(dev)) + dma_sync_virt(vaddr, size, direction); +} diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c index 1986e09fb457..1601d90b087b 100644 --- a/arch/mips/mm/ioremap.c +++ b/arch/mips/mm/ioremap.c @@ -9,6 +9,7 @@ #include <linux/export.h> #include <asm/addrspace.h> #include <asm/byteorder.h> +#include <linux/ioport.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -98,6 +99,20 @@ static int remap_area_pages(unsigned long address, phys_addr_t phys_addr, return error; } +static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages, + void *arg) +{ + unsigned long i; + + for (i = 0; i < nr_pages; i++) { + if (pfn_valid(start_pfn + i) && + !PageReserved(pfn_to_page(start_pfn + i))) + return 1; + } + + return 0; +} + /* * Generic mapping function (not visible outside): */ @@ -116,8 +131,8 @@ static int remap_area_pages(unsigned long address, phys_addr_t phys_addr, void __iomem * __ioremap(phys_addr_t phys_addr, phys_addr_t size, unsigned long flags) { + unsigned long offset, pfn, last_pfn; struct vm_struct * area; - unsigned long offset; phys_addr_t last_addr; void * addr; @@ -137,18 +152,16 @@ void __iomem * __ioremap(phys_addr_t phys_addr, phys_addr_t size, unsigned long return (void __iomem *) CKSEG1ADDR(phys_addr); /* - * Don't allow anybody to remap normal RAM that we're using.. + * Don't allow anybody to remap RAM that may be allocated by the page + * allocator, since that could lead to races & data clobbering. */ - if (phys_addr < virt_to_phys(high_memory)) { - char *t_addr, *t_end; - struct page *page; - - t_addr = __va(phys_addr); - t_end = t_addr + (size - 1); - - for(page = virt_to_page(t_addr); page <= virt_to_page(t_end); page++) - if(!PageReserved(page)) - return NULL; + pfn = PFN_DOWN(phys_addr); + last_pfn = PFN_DOWN(last_addr); + if (walk_system_ram_range(pfn, last_pfn - pfn + 1, NULL, + __ioremap_check_ram) == 1) { + WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n", + &phys_addr, &last_addr); + return NULL; } /* diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c index d5d02993aa21..56e4f8bffd4c 100644 --- a/arch/mips/mm/page.c +++ b/arch/mips/mm/page.c @@ -623,21 +623,6 @@ struct dmadscr { u64 pad_b; } ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS]; -void sb1_dma_init(void) -{ - int i; - - for (i = 0; i < DM_NUM_CHANNELS; i++) { - const u64 base_val = CPHYSADDR((unsigned long)&page_descr[i]) | - V_DM_DSCR_BASE_RINGSZ(1); - void *base_reg = IOADDR(A_DM_REGISTER(i, R_DM_DSCR_BASE)); - - __raw_writeq(base_val, base_reg); - __raw_writeq(base_val | M_DM_DSCR_BASE_RESET, base_reg); - __raw_writeq(base_val | M_DM_DSCR_BASE_ENABL, base_reg); - } -} - void clear_page(void *page) { u64 to_phys = CPHYSADDR((unsigned long)page); diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 79b9f2ad3ff5..49312a14cd17 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -1509,7 +1509,7 @@ static void setup_pw(void) #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT write_c0_pwctl(1 << 6 | psn); #endif - write_c0_kpgd(swapper_pg_dir); + write_c0_kpgd((long)swapper_pg_dir); kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */ } diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c index 9bb6baa45da3..24e5b0d06899 100644 --- a/arch/mips/mm/uasm-micromips.c +++ b/arch/mips/mm/uasm-micromips.c @@ -19,7 +19,6 @@ #include <asm/inst.h> #include <asm/elf.h> #include <asm/bugs.h> -#define UASM_ISA _UASM_ISA_MICROMIPS #include <asm/uasm.h> #define RS_MASK 0x1f diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index 9fea6c6bbf49..60ceb93c71a0 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -19,7 +19,6 @@ #include <asm/inst.h> #include <asm/elf.h> #include <asm/bugs.h> -#define UASM_ISA _UASM_ISA_CLASSIC #include <asm/uasm.h> #define RS_MASK 0x1f diff --git a/arch/mips/mti-malta/Makefile b/arch/mips/mti-malta/Makefile index 63940bdce698..17c7fd471a27 100644 --- a/arch/mips/mti-malta/Makefile +++ b/arch/mips/mti-malta/Makefile @@ -13,11 +13,9 @@ obj-y += malta-init.o obj-y += malta-int.o obj-y += malta-memory.o obj-y += malta-platform.o -obj-y += malta-reset.o obj-y += malta-setup.o obj-y += malta-time.o obj-$(CONFIG_MIPS_CMP) += malta-amon.o -obj-$(CONFIG_MIPS_MALTA_PM) += malta-pm.o CFLAGS_malta-dtshim.o = -I$(src)/../../../scripts/dtc/libfdt diff --git a/arch/mips/mti-malta/malta-pm.c b/arch/mips/mti-malta/malta-pm.c deleted file mode 100644 index efbd659fb602..000000000000 --- a/arch/mips/mti-malta/malta-pm.c +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2014 Imagination Technologies - * Author: Paul Burton <paul.burton@mips.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/io.h> -#include <linux/pci.h> - -#include <asm/mach-malta/malta-pm.h> - -static struct pci_bus *pm_pci_bus; -static resource_size_t pm_io_offset; - -int mips_pm_suspend(unsigned state) -{ - int spec_devid; - u16 sts; - - if (!pm_pci_bus || !pm_io_offset) - return -ENODEV; - - /* Ensure the power button status is clear */ - while (1) { - sts = inw(pm_io_offset + PIIX4_FUNC3IO_PMSTS); - if (!(sts & PIIX4_FUNC3IO_PMSTS_PWRBTN_STS)) - break; - outw(sts, pm_io_offset + PIIX4_FUNC3IO_PMSTS); - } - - /* Enable entry to suspend */ - outw(state | PIIX4_FUNC3IO_PMCNTRL_SUS_EN, - pm_io_offset + PIIX4_FUNC3IO_PMCNTRL); - - /* If the special cycle occurs too soon this doesn't work... */ - mdelay(10); - - /* - * The PIIX4 will enter the suspend state only after seeing a special - * cycle with the correct magic data on the PCI bus. Generate that - * cycle now. - */ - spec_devid = PCI_DEVID(0, PCI_DEVFN(0x1f, 0x7)); - pci_bus_write_config_dword(pm_pci_bus, spec_devid, 0, - PIIX4_SUSPEND_MAGIC); - - /* Give the system some time to power down */ - mdelay(1000); - - return 0; -} - -static int __init malta_pm_setup(void) -{ - struct pci_dev *dev; - int res, io_region = PCI_BRIDGE_RESOURCES; - - /* Find a reference to the PCI bus */ - pm_pci_bus = pci_find_next_bus(NULL); - if (!pm_pci_bus) { - pr_warn("malta-pm: failed to find reference to PCI bus\n"); - return -ENODEV; - } - - /* Find the PIIX4 PM device */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, - PCI_DEVICE_ID_INTEL_82371AB_3, PCI_ANY_ID, - PCI_ANY_ID, NULL); - if (!dev) { - pr_warn("malta-pm: failed to find PIIX4 PM\n"); - return -ENODEV; - } - - /* Request access to the PIIX4 PM IO registers */ - res = pci_request_region(dev, io_region, "PIIX4 PM IO registers"); - if (res) { - pr_warn("malta-pm: failed to request PM IO registers (%d)\n", - res); - pci_dev_put(dev); - return -ENODEV; - } - - /* Find the offset to the PIIX4 PM IO registers */ - pm_io_offset = pci_resource_start(dev, io_region); - - pci_dev_put(dev); - return 0; -} - -late_initcall(malta_pm_setup); diff --git a/arch/mips/mti-malta/malta-reset.c b/arch/mips/mti-malta/malta-reset.c deleted file mode 100644 index dd6f62ad4417..000000000000 --- a/arch/mips/mti-malta/malta-reset.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Carsten Langgaard, carstenl@mips.com - * Copyright (C) 1999,2000 MIPS Technologies, Inc. All rights reserved. - */ -#include <linux/io.h> -#include <linux/pm.h> -#include <linux/reboot.h> - -#include <asm/reboot.h> -#include <asm/mach-malta/malta-pm.h> - -static void mips_machine_power_off(void) -{ - mips_pm_suspend(PIIX4_FUNC3IO_PMCNTRL_SUS_TYP_SOFF); - - pr_info("Failed to power down, resetting\n"); - machine_restart(NULL); -} - -static int __init mips_reboot_setup(void) -{ - pm_power_off = mips_machine_power_off; - - return 0; -} -arch_initcall(mips_reboot_setup); diff --git a/arch/mips/mti-malta/malta-setup.c b/arch/mips/mti-malta/malta-setup.c index 7b63914d2e58..5d4c5e5fbd69 100644 --- a/arch/mips/mti-malta/malta-setup.c +++ b/arch/mips/mti-malta/malta-setup.c @@ -26,6 +26,7 @@ #include <linux/screen_info.h> #include <linux/time.h> +#include <asm/dma-coherence.h> #include <asm/fw/fw.h> #include <asm/mach-malta/malta-dtshim.h> #include <asm/mips-cps.h> @@ -144,12 +145,6 @@ static int __init plat_enable_iocoherency(void) static void __init plat_setup_iocoherency(void) { -#ifdef CONFIG_DMA_NONCOHERENT - /* - * Kernel has been configured with software coherency - * but we might choose to turn it off and use hardware - * coherency instead. - */ if (plat_enable_iocoherency()) { if (coherentio == IO_COHERENCE_DISABLED) pr_info("Hardware DMA cache coherency disabled\n"); @@ -161,10 +156,6 @@ static void __init plat_setup_iocoherency(void) else pr_info("Software DMA cache coherency enabled\n"); } -#else - if (!plat_enable_iocoherency()) - panic("Hardware DMA cache coherency not supported!"); -#endif } static void __init pci_clock_check(void) @@ -226,29 +217,6 @@ static void __init bonito_quirks_setup(void) pr_info("Enabled Bonito debug mode\n"); } else BONITO_BONGENCFG &= ~BONITO_BONGENCFG_DEBUGMODE; - -#ifdef CONFIG_DMA_COHERENT - if (BONITO_PCICACHECTRL & BONITO_PCICACHECTRL_CPUCOH_PRES) { - BONITO_PCICACHECTRL |= BONITO_PCICACHECTRL_CPUCOH_EN; - pr_info("Enabled Bonito CPU coherency\n"); - - argptr = fw_getcmdline(); - if (strstr(argptr, "iobcuncached")) { - BONITO_PCICACHECTRL &= ~BONITO_PCICACHECTRL_IOBCCOH_EN; - BONITO_PCIMEMBASECFG = BONITO_PCIMEMBASECFG & - ~(BONITO_PCIMEMBASECFG_MEMBASE0_CACHED | - BONITO_PCIMEMBASECFG_MEMBASE1_CACHED); - pr_info("Disabled Bonito IOBC coherency\n"); - } else { - BONITO_PCICACHECTRL |= BONITO_PCICACHECTRL_IOBCCOH_EN; - BONITO_PCIMEMBASECFG |= - (BONITO_PCIMEMBASECFG_MEMBASE0_CACHED | - BONITO_PCIMEMBASECFG_MEMBASE1_CACHED); - pr_info("Enabled Bonito IOBC coherency\n"); - } - } else - panic("Hardware DMA cache coherency not supported"); -#endif } void __init *plat_get_fdt(void) @@ -279,11 +247,6 @@ void __init plat_mem_setup(void) */ enable_dma(4); -#ifdef CONFIG_DMA_COHERENT - if (mips_revision_sconid != MIPS_REVISION_SCON_BONITO) - panic("Hardware DMA cache coherency not supported"); -#endif - if (mips_revision_sconid == MIPS_REVISION_SCON_BONITO) bonito_quirks_setup(); diff --git a/arch/mips/netlogic/common/earlycons.c b/arch/mips/netlogic/common/earlycons.c index 769f93032c53..8f5bc1597550 100644 --- a/arch/mips/netlogic/common/earlycons.c +++ b/arch/mips/netlogic/common/earlycons.c @@ -36,6 +36,7 @@ #include <linux/serial_reg.h> #include <asm/mipsregs.h> +#include <asm/setup.h> #include <asm/netlogic/haldefs.h> #include <asm/netlogic/common.h> diff --git a/arch/mips/netlogic/xlp/dt.c b/arch/mips/netlogic/xlp/dt.c index 856a6e6d296e..b5ba83f4c646 100644 --- a/arch/mips/netlogic/xlp/dt.c +++ b/arch/mips/netlogic/xlp/dt.c @@ -93,17 +93,3 @@ void __init device_tree_init(void) { unflatten_and_copy_device_tree(); } - -static struct of_device_id __initdata xlp_ids[] = { - { .compatible = "simple-bus", }, - {}, -}; - -int __init xlp8xx_ds_publish_devices(void) -{ - if (!of_have_populated_dt()) - return 0; - return of_platform_bus_probe(NULL, xlp_ids, NULL); -} - -device_initcall(xlp8xx_ds_publish_devices); diff --git a/arch/mips/paravirt/serial.c b/arch/mips/paravirt/serial.c index 02b665c02272..a37b6f9f0ede 100644 --- a/arch/mips/paravirt/serial.c +++ b/arch/mips/paravirt/serial.c @@ -9,16 +9,15 @@ #include <linux/kernel.h> #include <linux/virtio_console.h> #include <linux/kvm_para.h> +#include <asm/setup.h> /* * Emit one character to the boot console. */ -int prom_putchar(char c) +void prom_putchar(char c) { kvm_hypercall3(KVM_HC_MIPS_CONSOLE_OUTPUT, 0 /* port 0 */, (unsigned long)&c, 1 /* len == 1 */); - - return 1; } #ifdef CONFIG_VIRTIO_CONSOLE diff --git a/arch/mips/pci/ops-bridge.c b/arch/mips/pci/ops-bridge.c index 57e1463fcd02..a1d2c4ae0d1b 100644 --- a/arch/mips/pci/ops-bridge.c +++ b/arch/mips/pci/ops-bridge.c @@ -167,7 +167,7 @@ oh_my_gawd: static int pci_read_config(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 * value) { - if (bus->number > 0) + if (!pci_is_root_bus(bus)) return pci_conf1_read_config(bus, devfn, where, size, value); return pci_conf0_read_config(bus, devfn, where, size, value); @@ -310,7 +310,7 @@ oh_my_gawd: static int pci_write_config(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 value) { - if (bus->number > 0) + if (!pci_is_root_bus(bus)) return pci_conf1_write_config(bus, devfn, where, size, value); return pci_conf0_write_config(bus, devfn, where, size, value); diff --git a/arch/mips/pci/pci-ar2315.c b/arch/mips/pci/pci-ar2315.c index b4fa6413c4e5..c539d0d2b0cf 100644 --- a/arch/mips/pci/pci-ar2315.c +++ b/arch/mips/pci/pci-ar2315.c @@ -149,6 +149,13 @@ #define AR2315_PCI_HOST_SLOT 3 #define AR2315_PCI_HOST_DEVID ((0xff18 << 16) | PCI_VENDOR_ID_ATHEROS) +/* + * We need some arbitrary non-zero value to be programmed to the BAR1 register + * of PCI host controller to enable DMA. The same value should be used as the + * offset to calculate the physical address of DMA buffer for PCI devices. + */ +#define AR2315_PCI_HOST_SDRAM_BASEADDR 0x20000000 + /* ??? access BAR */ #define AR2315_PCI_HOST_MBAR0 0x10000000 /* RAM access BAR */ @@ -167,6 +174,23 @@ struct ar2315_pci_ctrl { struct resource io_res; }; +static inline dma_addr_t ar2315_dev_offset(struct device *dev) +{ + if (dev && dev_is_pci(dev)) + return AR2315_PCI_HOST_SDRAM_BASEADDR; + return 0; +} + +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + return paddr + ar2315_dev_offset(dev); +} + +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) +{ + return dma_addr - ar2315_dev_offset(dev); +} + static inline struct ar2315_pci_ctrl *ar2315_pci_bus_to_apc(struct pci_bus *bus) { struct pci_controller *hose = bus->sysdata; diff --git a/arch/mips/pci/pci-ar724x.c b/arch/mips/pci/pci-ar724x.c index 1e23c8d587bd..64b58cc48a91 100644 --- a/arch/mips/pci/pci-ar724x.c +++ b/arch/mips/pci/pci-ar724x.c @@ -12,14 +12,18 @@ #include <linux/irq.h> #include <linux/pci.h> #include <linux/init.h> +#include <linux/delay.h> #include <linux/platform_device.h> #include <asm/mach-ath79/ath79.h> #include <asm/mach-ath79/ar71xx_regs.h> +#define AR724X_PCI_REG_APP 0x00 #define AR724X_PCI_REG_RESET 0x18 #define AR724X_PCI_REG_INT_STATUS 0x4c #define AR724X_PCI_REG_INT_MASK 0x50 +#define AR724X_PCI_APP_LTSSM_ENABLE BIT(0) + #define AR724X_PCI_RESET_LINK_UP BIT(0) #define AR724X_PCI_INT_DEV0 BIT(14) @@ -325,6 +329,37 @@ static void ar724x_pci_irq_init(struct ar724x_pci_controller *apc, apc); } +static void ar724x_pci_hw_init(struct ar724x_pci_controller *apc) +{ + u32 ppl, app; + int wait = 0; + + /* deassert PCIe host controller and PCIe PHY reset */ + ath79_device_reset_clear(AR724X_RESET_PCIE); + ath79_device_reset_clear(AR724X_RESET_PCIE_PHY); + + /* remove the reset of the PCIE PLL */ + ppl = ath79_pll_rr(AR724X_PLL_REG_PCIE_CONFIG); + ppl &= ~AR724X_PLL_REG_PCIE_CONFIG_PPL_RESET; + ath79_pll_wr(AR724X_PLL_REG_PCIE_CONFIG, ppl); + + /* deassert bypass for the PCIE PLL */ + ppl = ath79_pll_rr(AR724X_PLL_REG_PCIE_CONFIG); + ppl &= ~AR724X_PLL_REG_PCIE_CONFIG_PPL_BYPASS; + ath79_pll_wr(AR724X_PLL_REG_PCIE_CONFIG, ppl); + + /* set PCIE Application Control to ready */ + app = __raw_readl(apc->ctrl_base + AR724X_PCI_REG_APP); + app |= AR724X_PCI_APP_LTSSM_ENABLE; + __raw_writel(app, apc->ctrl_base + AR724X_PCI_REG_APP); + + /* wait up to 100ms for PHY link up */ + do { + mdelay(10); + wait++; + } while (wait < 10 && !ar724x_pci_check_link(apc)); +} + static int ar724x_pci_probe(struct platform_device *pdev) { struct ar724x_pci_controller *apc; @@ -383,6 +418,13 @@ static int ar724x_pci_probe(struct platform_device *pdev) apc->pci_controller.io_resource = &apc->io_res; apc->pci_controller.mem_resource = &apc->mem_res; + /* + * Do the full PCIE Root Complex Initialization Sequence if the PCIe + * host controller is in reset. + */ + if (ath79_reset_rr(AR724X_RESET_REG_RESET_MODULE) & AR724X_RESET_PCIE) + ar724x_pci_hw_init(apc); + apc->link_up = ar724x_pci_check_link(apc); if (!apc->link_up) dev_warn(&pdev->dev, "PCIe link is down\n"); diff --git a/arch/mips/pci/pci-ip27.c b/arch/mips/pci/pci-ip27.c index 0f09eafa5e3a..c94a66070a60 100644 --- a/arch/mips/pci/pci-ip27.c +++ b/arch/mips/pci/pci-ip27.c @@ -11,6 +11,7 @@ #include <linux/export.h> #include <linux/pci.h> #include <linux/smp.h> +#include <linux/dma-direct.h> #include <asm/sn/arch.h> #include <asm/pci/bridge.h> #include <asm/paccess.h> @@ -182,6 +183,19 @@ int pcibios_plat_dev_init(struct pci_dev *dev) return 0; } +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct bridge_controller *bc = BRIDGE_CONTROLLER(pdev->bus); + + return bc->baddr + paddr; +} + +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) +{ + return dma_addr & ~(0xffUL << 56); +} + /* * Device might live on a subordinate PCI bus. XXX Walk up the chain of buses * to find the slot number in sense of the bridge device register. @@ -200,17 +214,6 @@ static inline void pci_disable_swapping(struct pci_dev *dev) bridge->b_widget.w_tflush; /* Flush */ } -static inline void pci_enable_swapping(struct pci_dev *dev) -{ - struct bridge_controller *bc = BRIDGE_CONTROLLER(dev->bus); - bridge_t *bridge = bc->base; - int slot = PCI_SLOT(dev->devfn); - - /* Turn on byte swapping */ - bridge->b_device[slot].reg |= BRIDGE_DEV_SWAP_DIR; - bridge->b_widget.w_tflush; /* Flush */ -} - static void pci_fixup_ioc3(struct pci_dev *d) { pci_disable_swapping(d); diff --git a/arch/mips/pci/pci-octeon.c b/arch/mips/pci/pci-octeon.c index 3e92a06fa772..5017d5843c5a 100644 --- a/arch/mips/pci/pci-octeon.c +++ b/arch/mips/pci/pci-octeon.c @@ -21,8 +21,6 @@ #include <asm/octeon/cvmx-pci-defs.h> #include <asm/octeon/pci-octeon.h> -#include <dma-coherence.h> - #define USE_OCTEON_INTERNAL_ARBITER /* @@ -166,8 +164,6 @@ int pcibios_plat_dev_init(struct pci_dev *dev) pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, dconfig); } - dev->dev.dma_ops = octeon_pci_dma_map_ops; - return 0; } diff --git a/arch/mips/pci/pci.c b/arch/mips/pci/pci.c index 9632436d74d7..c2e94cf5ecda 100644 --- a/arch/mips/pci/pci.c +++ b/arch/mips/pci/pci.c @@ -54,5 +54,5 @@ void pci_resource_to_user(const struct pci_dev *dev, int bar, phys_addr_t size = resource_size(rsrc); *start = fixup_bigphys_addr(rsrc->start, size); - *end = rsrc->start + size; + *end = rsrc->start + size - 1; } diff --git a/arch/mips/pci/pcie-octeon.c b/arch/mips/pci/pcie-octeon.c index 87ba86bd8696..d919a0d813a1 100644 --- a/arch/mips/pci/pcie-octeon.c +++ b/arch/mips/pci/pcie-octeon.c @@ -94,8 +94,6 @@ union cvmx_pcie_address { static int cvmx_pcie_rc_initialize(int pcie_port); -#include <dma-coherence.h> - /** * Return the Core virtual base address for PCIe IO access. IOs are * read/written as an offset from this address. @@ -1239,14 +1237,14 @@ static int __cvmx_pcie_rc_initialize_gen2(int pcie_port) /* CN63XX Pass 1.0 errata G-14395 requires the QLM De-emphasis be programmed */ if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_0)) { if (pcie_port) { - union cvmx_ciu_qlm1 ciu_qlm; + union cvmx_ciu_qlm ciu_qlm; ciu_qlm.u64 = cvmx_read_csr(CVMX_CIU_QLM1); ciu_qlm.s.txbypass = 1; ciu_qlm.s.txdeemph = 5; ciu_qlm.s.txmargin = 0x17; cvmx_write_csr(CVMX_CIU_QLM1, ciu_qlm.u64); } else { - union cvmx_ciu_qlm0 ciu_qlm; + union cvmx_ciu_qlm ciu_qlm; ciu_qlm.u64 = cvmx_read_csr(CVMX_CIU_QLM0); ciu_qlm.s.txbypass = 1; ciu_qlm.s.txdeemph = 5; diff --git a/arch/mips/pic32/pic32mzda/early_console.c b/arch/mips/pic32/pic32mzda/early_console.c index d7b783463fac..8ed4961b1271 100644 --- a/arch/mips/pic32/pic32mzda/early_console.c +++ b/arch/mips/pic32/pic32mzda/early_console.c @@ -13,6 +13,7 @@ */ #include <asm/mach-pic32/pic32.h> #include <asm/fw/fw.h> +#include <asm/setup.h> #include "pic32mzda.h" #include "early_pin.h" @@ -157,7 +158,7 @@ void __init fw_init_early_console(char port) setup_early_console(port, baud); } -int prom_putchar(char c) +void prom_putchar(char c) { if (console_port >= 0) { while (__raw_readl( @@ -166,6 +167,4 @@ int prom_putchar(char c) __raw_writel(c, uart_base + U_TXR(console_port)); } - - return 1; } diff --git a/arch/mips/ralink/early_printk.c b/arch/mips/ralink/early_printk.c index 3c59ffe5f5f5..ecd30ddfb3db 100644 --- a/arch/mips/ralink/early_printk.c +++ b/arch/mips/ralink/early_printk.c @@ -10,6 +10,7 @@ #include <linux/serial_reg.h> #include <asm/addrspace.h> +#include <asm/setup.h> #ifdef CONFIG_SOC_RT288X #define EARLY_UART_BASE 0x300c00 @@ -68,7 +69,7 @@ static void find_uart_base(void) } } -void prom_putchar(unsigned char ch) +void prom_putchar(char ch) { if (!init_complete) { find_uart_base(); @@ -76,13 +77,13 @@ void prom_putchar(unsigned char ch) } if (IS_ENABLED(CONFIG_SOC_MT7621) || soc_is_mt7628()) { - uart_w32(ch, UART_TX); + uart_w32((unsigned char)ch, UART_TX); while ((uart_r32(UART_REG_LSR) & UART_LSR_THRE) == 0) ; } else { while ((uart_r32(UART_REG_LSR_RT2880) & UART_LSR_THRE) == 0) ; - uart_w32(ch, UART_REG_TX); + uart_w32((unsigned char)ch, UART_REG_TX); while ((uart_r32(UART_REG_LSR_RT2880) & UART_LSR_THRE) == 0) ; } diff --git a/arch/mips/sgi-ip27/ip27-console.c b/arch/mips/sgi-ip27/ip27-console.c index 45fdfbcbd4c6..6bdb48d41276 100644 --- a/arch/mips/sgi-ip27/ip27-console.c +++ b/arch/mips/sgi-ip27/ip27-console.c @@ -7,6 +7,7 @@ */ #include <asm/page.h> +#include <asm/setup.h> #include <asm/sn/addrs.h> #include <asm/sn/sn0/hub.h> #include <asm/sn/klconfig.h> diff --git a/arch/mips/sgi-ip32/Makefile b/arch/mips/sgi-ip32/Makefile index 60f0227425e7..4745cd94df11 100644 --- a/arch/mips/sgi-ip32/Makefile +++ b/arch/mips/sgi-ip32/Makefile @@ -4,4 +4,4 @@ # obj-y += ip32-berr.o ip32-irq.o ip32-platform.o ip32-setup.o ip32-reset.o \ - crime.o ip32-memory.o + crime.o ip32-memory.o ip32-dma.o diff --git a/arch/mips/sgi-ip32/ip32-dma.c b/arch/mips/sgi-ip32/ip32-dma.c new file mode 100644 index 000000000000..fa7b17cb5385 --- /dev/null +++ b/arch/mips/sgi-ip32/ip32-dma.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2006 Ralf Baechle <ralf@linux-mips.org> + */ +#include <linux/dma-direct.h> +#include <asm/ip32/crime.h> + +/* + * Few notes. + * 1. CPU sees memory as two chunks: 0-256M@0x0, and the rest @0x40000000+256M + * 2. PCI sees memory as one big chunk @0x0 (or we could use 0x40000000 for + * native-endian) + * 3. All other devices see memory as one big chunk at 0x40000000 + * 4. Non-PCI devices will pass NULL as struct device* + * + * Thus we translate differently, depending on device. + */ + +#define RAM_OFFSET_MASK 0x3fffffffUL + +dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + dma_addr_t dma_addr = paddr & RAM_OFFSET_MASK; + + if (!dev) + dma_addr += CRIME_HI_MEM_BASE; + return dma_addr; +} + +phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dma_addr) +{ + phys_addr_t paddr = dma_addr & RAM_OFFSET_MASK; + + if (dma_addr >= 256*1024*1024) + paddr += CRIME_HI_MEM_BASE; + return paddr; +} diff --git a/arch/mips/sibyte/Kconfig b/arch/mips/sibyte/Kconfig index f4dbce25bc6a..7ec278d72096 100644 --- a/arch/mips/sibyte/Kconfig +++ b/arch/mips/sibyte/Kconfig @@ -70,7 +70,6 @@ config SIBYTE_BCM1x55 config SIBYTE_SB1xxx_SOC bool - select DMA_COHERENT select IRQ_MIPS_CPU select SWAP_IO_SPACE select SYS_SUPPORTS_32BIT_KERNEL diff --git a/arch/mips/sibyte/common/cfe.c b/arch/mips/sibyte/common/cfe.c index 115399202eab..092fb2a6ec4a 100644 --- a/arch/mips/sibyte/common/cfe.c +++ b/arch/mips/sibyte/common/cfe.c @@ -27,6 +27,7 @@ #include <asm/bootinfo.h> #include <asm/reboot.h> +#include <asm/setup.h> #include <asm/sibyte/board.h> #include <asm/smp-ops.h> diff --git a/arch/mips/txx9/generic/setup.c b/arch/mips/txx9/generic/setup.c index aa47932abd28..f6d9182ef82a 100644 --- a/arch/mips/txx9/generic/setup.c +++ b/arch/mips/txx9/generic/setup.c @@ -33,6 +33,7 @@ #include <asm/reboot.h> #include <asm/r4kcache.h> #include <asm/sections.h> +#include <asm/setup.h> #include <asm/txx9/generic.h> #include <asm/txx9/pci.h> #include <asm/txx9tmr.h> diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index ce196046ac3e..34605ca21498 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -7,7 +7,13 @@ ccflags-vdso := \ $(filter -I%,$(KBUILD_CFLAGS)) \ $(filter -E%,$(KBUILD_CFLAGS)) \ $(filter -mmicromips,$(KBUILD_CFLAGS)) \ - $(filter -march=%,$(KBUILD_CFLAGS)) + $(filter -march=%,$(KBUILD_CFLAGS)) \ + -D__VDSO__ + +ifeq ($(cc-name),clang) +ccflags-vdso += $(filter --target=%,$(KBUILD_CFLAGS)) +endif + cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ -O2 -g -fPIC -fno-strict-aliasing -fno-common -fno-builtin -G 0 \ @@ -38,6 +44,7 @@ endif # VDSO linker flags. VDSO_LDFLAGS := \ -Wl,-Bsymbolic -Wl,--no-undefined -Wl,-soname=linux-vdso.so.1 \ + $(addprefix -Wl$(comma),$(filter -E%,$(KBUILD_CFLAGS))) \ -nostdlib -shared \ $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) \ $(call cc-ldoption, -Wl$(comma)--build-id) diff --git a/arch/mips/vdso/genvdso.h b/arch/mips/vdso/genvdso.h index 94334727059a..611b06f01a3c 100644 --- a/arch/mips/vdso/genvdso.h +++ b/arch/mips/vdso/genvdso.h @@ -15,8 +15,6 @@ static inline bool FUNC(patch_vdso)(const char *path, void *vdso) ELF(Shdr) *shdr; char *shstrtab, *name; uint16_t sh_count, sh_entsize, i; - unsigned int local_gotno, symtabno, gotsym; - ELF(Dyn) *dyn = NULL; shdrs = vdso + FUNC(swap_uint)(ehdr->e_shoff); sh_count = swap_uint16(ehdr->e_shnum); @@ -41,9 +39,6 @@ static inline bool FUNC(patch_vdso)(const char *path, void *vdso) "%s: '%s' contains relocation sections\n", program_name, path); return false; - case SHT_DYNAMIC: - dyn = vdso + FUNC(swap_uint)(shdr->sh_offset); - break; } /* Check for existing sections. */ @@ -61,52 +56,6 @@ static inline bool FUNC(patch_vdso)(const char *path, void *vdso) } } - /* - * Ensure the GOT has no entries other than the standard 2, for the same - * reason we check that there's no relocation sections above. - * The standard two entries are: - * - Lazy resolver - * - Module pointer - */ - if (dyn) { - local_gotno = symtabno = gotsym = 0; - - while (FUNC(swap_uint)(dyn->d_tag) != DT_NULL) { - switch (FUNC(swap_uint)(dyn->d_tag)) { - /* - * This member holds the number of local GOT entries. - */ - case DT_MIPS_LOCAL_GOTNO: - local_gotno = FUNC(swap_uint)(dyn->d_un.d_val); - break; - /* - * This member holds the number of entries in the - * .dynsym section. - */ - case DT_MIPS_SYMTABNO: - symtabno = FUNC(swap_uint)(dyn->d_un.d_val); - break; - /* - * This member holds the index of the first dynamic - * symbol table entry that corresponds to an entry in - * the GOT. - */ - case DT_MIPS_GOTSYM: - gotsym = FUNC(swap_uint)(dyn->d_un.d_val); - break; - } - - dyn++; - } - - if (local_gotno > 2 || symtabno - gotsym) { - fprintf(stderr, - "%s: '%s' contains unexpected GOT entries\n", - program_name, path); - return false; - } - } - return true; } diff --git a/arch/mips/vr41xx/common/pmu.c b/arch/mips/vr41xx/common/pmu.c index 39a0db3e2b34..16e684b59875 100644 --- a/arch/mips/vr41xx/common/pmu.c +++ b/arch/mips/vr41xx/common/pmu.c @@ -17,6 +17,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include <linux/cpu.h> #include <linux/errno.h> #include <linux/init.h> #include <linux/ioport.h> @@ -46,7 +47,7 @@ static void __iomem *pmu_base; #define pmu_read(offset) readw(pmu_base + (offset)) #define pmu_write(offset, value) writew((value), pmu_base + (offset)) -static void vr41xx_cpu_wait(void) +static void __cpuidle vr41xx_cpu_wait(void) { local_irq_disable(); if (!need_resched()) diff --git a/arch/nds32/Kconfig b/arch/nds32/Kconfig index 6aed974276d8..34f7222c5efe 100644 --- a/arch/nds32/Kconfig +++ b/arch/nds32/Kconfig @@ -12,17 +12,17 @@ config NDS32 select CLONE_BACKWARDS select COMMON_CLK select DMA_NONCOHERENT_OPS - select GENERIC_ASHLDI3 - select GENERIC_ASHRDI3 - select GENERIC_LSHRDI3 - select GENERIC_CMPDI2 - select GENERIC_MULDI3 - select GENERIC_UCMPDI2 select GENERIC_ATOMIC64 select GENERIC_CPU_DEVICES select GENERIC_CLOCKEVENTS select GENERIC_IRQ_CHIP select GENERIC_IRQ_SHOW + select GENERIC_LIB_ASHLDI3 + select GENERIC_LIB_ASHRDI3 + select GENERIC_LIB_CMPDI2 + select GENERIC_LIB_LSHRDI3 + select GENERIC_LIB_MULDI3 + select GENERIC_LIB_UCMPDI2 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL diff --git a/arch/nds32/Makefile b/arch/nds32/Makefile index 513bb2e9baf9..031c676821ff 100644 --- a/arch/nds32/Makefile +++ b/arch/nds32/Makefile @@ -34,10 +34,12 @@ ifdef CONFIG_CPU_LITTLE_ENDIAN KBUILD_CFLAGS += $(call cc-option, -EL) KBUILD_AFLAGS += $(call cc-option, -EL) LDFLAGS += $(call cc-option, -EL) +CHECKFLAGS += -D__NDS32_EL__ else KBUILD_CFLAGS += $(call cc-option, -EB) KBUILD_AFLAGS += $(call cc-option, -EB) LDFLAGS += $(call cc-option, -EB) +CHECKFLAGS += -D__NDS32_EB__ endif boot := arch/nds32/boot diff --git a/arch/nds32/include/asm/cacheflush.h b/arch/nds32/include/asm/cacheflush.h index 10b48f0d8e85..8b26198d51bb 100644 --- a/arch/nds32/include/asm/cacheflush.h +++ b/arch/nds32/include/asm/cacheflush.h @@ -8,6 +8,8 @@ #define PG_dcache_dirty PG_arch_1 +void flush_icache_range(unsigned long start, unsigned long end); +void flush_icache_page(struct vm_area_struct *vma, struct page *page); #ifdef CONFIG_CPU_CACHE_ALIASING void flush_cache_mm(struct mm_struct *mm); void flush_cache_dup_mm(struct mm_struct *mm); @@ -34,13 +36,16 @@ void flush_anon_page(struct vm_area_struct *vma, void flush_kernel_dcache_page(struct page *page); void flush_kernel_vmap_range(void *addr, int size); void invalidate_kernel_vmap_range(void *addr, int size); -void flush_icache_range(unsigned long start, unsigned long end); -void flush_icache_page(struct vm_area_struct *vma, struct page *page); #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&(mapping)->i_pages) #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&(mapping)->i_pages) #else #include <asm-generic/cacheflush.h> +#undef flush_icache_range +#undef flush_icache_page +#undef flush_icache_user_range +void flush_icache_user_range(struct vm_area_struct *vma, struct page *page, + unsigned long addr, int len); #endif #endif /* __NDS32_CACHEFLUSH_H__ */ diff --git a/arch/nds32/include/asm/futex.h b/arch/nds32/include/asm/futex.h index eab5e84bd991..cb6cb91cfdf8 100644 --- a/arch/nds32/include/asm/futex.h +++ b/arch/nds32/include/asm/futex.h @@ -16,7 +16,7 @@ " .popsection\n" \ " .pushsection .fixup,\"ax\"\n" \ "4: move %0, " err_reg "\n" \ - " j 3b\n" \ + " b 3b\n" \ " .popsection" #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c index 2f5b2ccebe47..63a1a5ef5219 100644 --- a/arch/nds32/kernel/setup.c +++ b/arch/nds32/kernel/setup.c @@ -278,7 +278,8 @@ static void __init setup_memory(void) void __init setup_arch(char **cmdline_p) { - early_init_devtree( __dtb_start); + early_init_devtree(__atags_pointer ? \ + phys_to_virt(__atags_pointer) : __dtb_start); setup_cpuinfo(); diff --git a/arch/nds32/mm/cacheflush.c b/arch/nds32/mm/cacheflush.c index ce8fd34497bf..254703653b6f 100644 --- a/arch/nds32/mm/cacheflush.c +++ b/arch/nds32/mm/cacheflush.c @@ -13,7 +13,39 @@ extern struct cache_info L1_cache_info[2]; -#ifndef CONFIG_CPU_CACHE_ALIASING +void flush_icache_range(unsigned long start, unsigned long end) +{ + unsigned long line_size, flags; + line_size = L1_cache_info[DCACHE].line_size; + start = start & ~(line_size - 1); + end = (end + line_size - 1) & ~(line_size - 1); + local_irq_save(flags); + cpu_cache_wbinval_range(start, end, 1); + local_irq_restore(flags); +} +EXPORT_SYMBOL(flush_icache_range); + +void flush_icache_page(struct vm_area_struct *vma, struct page *page) +{ + unsigned long flags; + unsigned long kaddr; + local_irq_save(flags); + kaddr = (unsigned long)kmap_atomic(page); + cpu_cache_wbinval_page(kaddr, vma->vm_flags & VM_EXEC); + kunmap_atomic((void *)kaddr); + local_irq_restore(flags); +} +EXPORT_SYMBOL(flush_icache_page); + +void flush_icache_user_range(struct vm_area_struct *vma, struct page *page, + unsigned long addr, int len) +{ + unsigned long kaddr; + kaddr = (unsigned long)kmap_atomic(page) + (addr & ~PAGE_MASK); + flush_icache_range(kaddr, kaddr + len); + kunmap_atomic((void *)kaddr); +} + void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t * pte) { @@ -35,19 +67,15 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, if ((test_and_clear_bit(PG_dcache_dirty, &page->flags)) || (vma->vm_flags & VM_EXEC)) { - - if (!PageHighMem(page)) { - cpu_cache_wbinval_page((unsigned long) - page_address(page), - vma->vm_flags & VM_EXEC); - } else { - unsigned long kaddr = (unsigned long)kmap_atomic(page); - cpu_cache_wbinval_page(kaddr, vma->vm_flags & VM_EXEC); - kunmap_atomic((void *)kaddr); - } + unsigned long kaddr; + local_irq_save(flags); + kaddr = (unsigned long)kmap_atomic(page); + cpu_cache_wbinval_page(kaddr, vma->vm_flags & VM_EXEC); + kunmap_atomic((void *)kaddr); + local_irq_restore(flags); } } -#else +#ifdef CONFIG_CPU_CACHE_ALIASING extern pte_t va_present(struct mm_struct *mm, unsigned long addr); static inline unsigned long aliasing(unsigned long addr, unsigned long page) @@ -317,52 +345,4 @@ void invalidate_kernel_vmap_range(void *addr, int size) local_irq_restore(flags); } EXPORT_SYMBOL(invalidate_kernel_vmap_range); - -void flush_icache_range(unsigned long start, unsigned long end) -{ - unsigned long line_size, flags; - line_size = L1_cache_info[DCACHE].line_size; - start = start & ~(line_size - 1); - end = (end + line_size - 1) & ~(line_size - 1); - local_irq_save(flags); - cpu_cache_wbinval_range(start, end, 1); - local_irq_restore(flags); -} -EXPORT_SYMBOL(flush_icache_range); - -void flush_icache_page(struct vm_area_struct *vma, struct page *page) -{ - unsigned long flags; - local_irq_save(flags); - cpu_cache_wbinval_page((unsigned long)page_address(page), - vma->vm_flags & VM_EXEC); - local_irq_restore(flags); -} - -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, - pte_t * pte) -{ - struct page *page; - unsigned long flags; - unsigned long pfn = pte_pfn(*pte); - - if (!pfn_valid(pfn)) - return; - - if (vma->vm_mm == current->active_mm) { - local_irq_save(flags); - __nds32__mtsr_dsb(addr, NDS32_SR_TLB_VPN); - __nds32__tlbop_rwr(*pte); - __nds32__isb(); - local_irq_restore(flags); - } - - page = pfn_to_page(pfn); - if (test_and_clear_bit(PG_dcache_dirty, &page->flags) || - (vma->vm_flags & VM_EXEC)) { - local_irq_save(flags); - cpu_dcache_wbinval_page((unsigned long)page_address(page)); - local_irq_restore(flags); - } -} #endif diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 9ecad05bfc73..dfb6a79ba7ff 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -27,7 +27,6 @@ config OPENRISC select GENERIC_STRNLEN_USER select GENERIC_SMP_IDLE_THREAD select MODULES_USE_ELF_RELA - select MULTI_IRQ_HANDLER select HAVE_DEBUG_STACKOVERFLOW select OR1K_PIC select CPU_NO_EFFICIENT_FFS if !OPENRISC_HAVE_INST_FF1 @@ -36,6 +35,7 @@ config OPENRISC select ARCH_USE_QUEUED_RWLOCKS select OMPIC if SMP select ARCH_WANT_FRAME_POINTERS + select GENERIC_IRQ_MULTI_HANDLER config CPU_BIG_ENDIAN def_bool y @@ -69,9 +69,6 @@ config STACKTRACE_SUPPORT config LOCKDEP_SUPPORT def_bool y -config MULTI_IRQ_HANDLER - def_bool y - source "init/Kconfig" source "kernel/Kconfig.freezer" diff --git a/arch/openrisc/include/asm/atomic.h b/arch/openrisc/include/asm/atomic.h index 146e1660f00e..b589fac39b92 100644 --- a/arch/openrisc/include/asm/atomic.h +++ b/arch/openrisc/include/asm/atomic.h @@ -100,7 +100,7 @@ ATOMIC_OP(xor) * * This is often used through atomic_inc_not_zero() */ -static inline int __atomic_add_unless(atomic_t *v, int a, int u) +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int old, tmp; @@ -119,7 +119,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return old; } -#define __atomic_add_unless __atomic_add_unless +#define atomic_fetch_add_unless atomic_fetch_add_unless #include <asm-generic/atomic.h> diff --git a/arch/openrisc/include/asm/cmpxchg.h b/arch/openrisc/include/asm/cmpxchg.h index d29f7db53906..f9cd43a39d72 100644 --- a/arch/openrisc/include/asm/cmpxchg.h +++ b/arch/openrisc/include/asm/cmpxchg.h @@ -16,8 +16,9 @@ #ifndef __ASM_OPENRISC_CMPXCHG_H #define __ASM_OPENRISC_CMPXCHG_H +#include <linux/bits.h> +#include <linux/compiler.h> #include <linux/types.h> -#include <linux/bitops.h> #define __HAVE_ARCH_CMPXCHG 1 diff --git a/arch/openrisc/include/asm/irq.h b/arch/openrisc/include/asm/irq.h index d9eee0a2b7b4..eb612b1865d2 100644 --- a/arch/openrisc/include/asm/irq.h +++ b/arch/openrisc/include/asm/irq.h @@ -24,6 +24,4 @@ #define NO_IRQ (-1) -extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); - #endif /* __ASM_OPENRISC_IRQ_H__ */ diff --git a/arch/openrisc/include/asm/pgalloc.h b/arch/openrisc/include/asm/pgalloc.h index 3e1a46615120..8999b9226512 100644 --- a/arch/openrisc/include/asm/pgalloc.h +++ b/arch/openrisc/include/asm/pgalloc.h @@ -98,8 +98,12 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte) __free_page(pte); } +#define __pte_free_tlb(tlb, pte, addr) \ +do { \ + pgtable_page_dtor(pte); \ + tlb_remove_page((tlb), (pte)); \ +} while (0) -#define __pte_free_tlb(tlb, pte, addr) tlb_remove_page((tlb), (pte)) #define pmd_pgtable(pmd) pmd_page(pmd) #define check_pgt_cache() do { } while (0) diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S index 690d55272ba6..0c826ad6e994 100644 --- a/arch/openrisc/kernel/entry.S +++ b/arch/openrisc/kernel/entry.S @@ -277,12 +277,6 @@ EXCEPTION_ENTRY(_data_page_fault_handler) l.addi r3,r1,0 // pt_regs /* r4 set be EXCEPTION_HANDLE */ // effective address of fault - /* - * __PHX__: TODO - * - * all this can be written much simpler. look at - * DTLB miss handler in the CONFIG_GUARD_PROTECTED_CORE part - */ #ifdef CONFIG_OPENRISC_NO_SPR_SR_DSX l.lwz r6,PT_PC(r3) // address of an offending insn l.lwz r6,0(r6) // instruction that caused pf @@ -314,7 +308,7 @@ EXCEPTION_ENTRY(_data_page_fault_handler) #else - l.lwz r6,PT_SR(r3) // SR + l.mfspr r6,r0,SPR_SR // SR l.andi r6,r6,SPR_SR_DSX // check for delay slot exception l.sfne r6,r0 // exception happened in delay slot l.bnf 7f diff --git a/arch/openrisc/kernel/head.S b/arch/openrisc/kernel/head.S index fb02b2a1d6f2..9fc6b60140f0 100644 --- a/arch/openrisc/kernel/head.S +++ b/arch/openrisc/kernel/head.S @@ -210,8 +210,7 @@ * r4 - EEAR exception EA * r10 - current pointing to current_thread_info struct * r12 - syscall 0, since we didn't come from syscall - * r13 - temp it actually contains new SR, not needed anymore - * r31 - handler address of the handler we'll jump to + * r30 - handler address of the handler we'll jump to * * handler has to save remaining registers to the exception * ksp frame *before* tainting them! @@ -244,6 +243,7 @@ /* r1 is KSP, r30 is __pa(KSP) */ ;\ tophys (r30,r1) ;\ l.sw PT_GPR12(r30),r12 ;\ + /* r4 use for tmp before EA */ ;\ l.mfspr r12,r0,SPR_EPCR_BASE ;\ l.sw PT_PC(r30),r12 ;\ l.mfspr r12,r0,SPR_ESR_BASE ;\ @@ -263,7 +263,10 @@ /* r12 == 1 if we come from syscall */ ;\ CLEAR_GPR(r12) ;\ /* ----- turn on MMU ----- */ ;\ - l.ori r30,r0,(EXCEPTION_SR) ;\ + /* Carry DSX into exception SR */ ;\ + l.mfspr r30,r0,SPR_SR ;\ + l.andi r30,r30,SPR_SR_DSX ;\ + l.ori r30,r30,(EXCEPTION_SR) ;\ l.mtspr r0,r30,SPR_ESR_BASE ;\ /* r30: EA address of handler */ ;\ LOAD_SYMBOL_2_GPR(r30,handler) ;\ diff --git a/arch/openrisc/kernel/irq.c b/arch/openrisc/kernel/irq.c index 35e478a93116..5f9445effaf8 100644 --- a/arch/openrisc/kernel/irq.c +++ b/arch/openrisc/kernel/irq.c @@ -41,13 +41,6 @@ void __init init_IRQ(void) irqchip_init(); } -static void (*handle_arch_irq)(struct pt_regs *); - -void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) -{ - handle_arch_irq = handle_irq; -} - void __irq_entry do_IRQ(struct pt_regs *regs) { handle_arch_irq(regs); diff --git a/arch/openrisc/kernel/traps.c b/arch/openrisc/kernel/traps.c index fac246e6f37a..d8981cbb852a 100644 --- a/arch/openrisc/kernel/traps.c +++ b/arch/openrisc/kernel/traps.c @@ -300,7 +300,7 @@ static inline int in_delay_slot(struct pt_regs *regs) return 0; } #else - return regs->sr & SPR_SR_DSX; + return mfspr(SPR_SR) & SPR_SR_DSX; #endif } diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 17526bebcbd2..e21751fb24aa 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -11,7 +11,6 @@ config PARISC select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_STRICT_KERNEL_RWX select ARCH_HAS_UBSAN_SANITIZE_ALL - select ARCH_WANTS_UBSAN_NO_NULL select ARCH_SUPPORTS_MEMORY_FAILURE select RTC_CLASS select RTC_DRV_GENERIC @@ -46,6 +45,7 @@ config PARISC select HAVE_ARCH_HASH select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK + select HAVE_REGS_AND_STACK_ACCESS_API select GENERIC_SCHED_CLOCK select HAVE_UNSTABLE_SCHED_CLOCK if SMP select GENERIC_CLOCKEVENTS @@ -188,6 +188,10 @@ config PA20 config PA11 def_bool y depends on PA7000 || PA7100LC || PA7200 || PA7300LC + select ARCH_HAS_SYNC_DMA_FOR_CPU + select ARCH_HAS_SYNC_DMA_FOR_DEVICE + select DMA_NONCOHERENT_OPS + select DMA_NONCOHERENT_CACHE_SYNC config PREFETCH def_bool y @@ -195,7 +199,7 @@ config PREFETCH config MLONGCALLS bool "Enable the -mlong-calls compiler option for big kernels" - def_bool y if (!MODULES) + default y depends on PA8X00 help If you configure the kernel to include many drivers built-in instead diff --git a/arch/parisc/include/asm/assembly.h b/arch/parisc/include/asm/assembly.h index 60e6f07b7e32..e9c6385ef0d1 100644 --- a/arch/parisc/include/asm/assembly.h +++ b/arch/parisc/include/asm/assembly.h @@ -36,6 +36,7 @@ #define RP_OFFSET 16 #define FRAME_SIZE 128 #define CALLEE_REG_FRAME_SIZE 144 +#define REG_SZ 8 #define ASM_ULONG_INSN .dword #else /* CONFIG_64BIT */ #define LDREG ldw @@ -50,6 +51,7 @@ #define RP_OFFSET 20 #define FRAME_SIZE 64 #define CALLEE_REG_FRAME_SIZE 128 +#define REG_SZ 4 #define ASM_ULONG_INSN .word #endif diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 88bae6676c9b..118953d41763 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -77,30 +77,6 @@ static __inline__ int atomic_read(const atomic_t *v) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -/** - * __atomic_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - #define ATOMIC_OP(op, c_op) \ static __inline__ void atomic_##op(int i, atomic_t *v) \ { \ @@ -160,28 +136,6 @@ ATOMIC_OPS(xor, ^=) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_inc(v) (atomic_add( 1,(v))) -#define atomic_dec(v) (atomic_add( -1,(v))) - -#define atomic_inc_return(v) (atomic_add_return( 1,(v))) -#define atomic_dec_return(v) (atomic_add_return( -1,(v))) - -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) - -#define atomic_sub_and_test(i,v) (atomic_sub_return((i),(v)) == 0) - #define ATOMIC_INIT(i) { (i) } #ifdef CONFIG_64BIT @@ -264,72 +218,11 @@ atomic64_read(const atomic64_t *v) return READ_ONCE((v)->counter); } -#define atomic64_inc(v) (atomic64_add( 1,(v))) -#define atomic64_dec(v) (atomic64_add( -1,(v))) - -#define atomic64_inc_return(v) (atomic64_add_return( 1,(v))) -#define atomic64_dec_return(v) (atomic64_add_return( -1,(v))) - -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) - -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) -#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0) -#define atomic64_sub_and_test(i,v) (atomic64_sub_return((i),(v)) == 0) - /* exported interface */ #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -/** - * atomic64_add_unless - add unless the number is a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - c = atomic64_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic64_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c != (u); -} - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -/* - * atomic64_dec_if_positive - decrement by 1 if old value positive - * @v: pointer of type atomic_t - * - * The function returns the old value of *v minus 1, even if - * the atomic variable, v, was not decremented. - */ -static inline long atomic64_dec_if_positive(atomic64_t *v) -{ - long c, old, dec; - c = atomic64_read(v); - for (;;) { - dec = c - 1; - if (unlikely(dec < 0)) - break; - old = atomic64_cmpxchg((v), c, dec); - if (likely(old == c)) - break; - c = old; - } - return dec; -} - #endif /* !CONFIG_64BIT */ diff --git a/arch/parisc/include/asm/barrier.h b/arch/parisc/include/asm/barrier.h new file mode 100644 index 000000000000..dbaaca84f27f --- /dev/null +++ b/arch/parisc/include/asm/barrier.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_BARRIER_H +#define __ASM_BARRIER_H + +#ifndef __ASSEMBLY__ + +/* The synchronize caches instruction executes as a nop on systems in + which all memory references are performed in order. */ +#define synchronize_caches() __asm__ __volatile__ ("sync" : : : "memory") + +#if defined(CONFIG_SMP) +#define mb() do { synchronize_caches(); } while (0) +#define rmb() mb() +#define wmb() mb() +#define dma_rmb() mb() +#define dma_wmb() mb() +#else +#define mb() barrier() +#define rmb() barrier() +#define wmb() barrier() +#define dma_rmb() barrier() +#define dma_wmb() barrier() +#endif + +#define __smp_mb() mb() +#define __smp_rmb() mb() +#define __smp_wmb() mb() + +#include <asm-generic/barrier.h> + +#endif /* !__ASSEMBLY__ */ +#endif /* __ASM_BARRIER_H */ diff --git a/arch/parisc/include/asm/dma-mapping.h b/arch/parisc/include/asm/dma-mapping.h index 01e1fc057c83..44a9f97194aa 100644 --- a/arch/parisc/include/asm/dma-mapping.h +++ b/arch/parisc/include/asm/dma-mapping.h @@ -21,11 +21,6 @@ ** flush/purge and allocate "regular" cacheable pages for everything. */ -#ifdef CONFIG_PA11 -extern const struct dma_map_ops pcxl_dma_ops; -extern const struct dma_map_ops pcx_dma_ops; -#endif - extern const struct dma_map_ops *hppa_dma_ops; static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) diff --git a/arch/parisc/include/asm/linkage.h b/arch/parisc/include/asm/linkage.h index 9a69bf6fc4b6..49f6f3d772cc 100644 --- a/arch/parisc/include/asm/linkage.h +++ b/arch/parisc/include/asm/linkage.h @@ -18,9 +18,9 @@ #ifdef __ASSEMBLY__ #define ENTRY(name) \ - .export name !\ - ALIGN !\ -name: + ALIGN !\ +name: ASM_NL\ + .export name #ifdef CONFIG_64BIT #define ENDPROC(name) \ @@ -31,13 +31,18 @@ name: END(name) #endif -#define ENTRY_CFI(name) \ +#define ENTRY_CFI(name, ...) \ ENTRY(name) ASM_NL\ + .proc ASM_NL\ + .callinfo __VA_ARGS__ ASM_NL\ + .entry ASM_NL\ CFI_STARTPROC #define ENDPROC_CFI(name) \ - ENDPROC(name) ASM_NL\ - CFI_ENDPROC + CFI_ENDPROC ASM_NL\ + .exit ASM_NL\ + .procend ASM_NL\ + ENDPROC(name) #endif /* __ASSEMBLY__ */ diff --git a/arch/parisc/include/asm/ptrace.h b/arch/parisc/include/asm/ptrace.h index 46da07670c2b..2a27b275ab09 100644 --- a/arch/parisc/include/asm/ptrace.h +++ b/arch/parisc/include/asm/ptrace.h @@ -25,4 +25,15 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->gr[20]; } +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->iaoq[0] = val; +} + +/* Query offset/name of register from its name/offset */ +extern int regs_query_register_offset(const char *name); +extern const char *regs_query_register_name(unsigned int offset); +#define MAX_REG_OFFSET (offsetof(struct pt_regs, ipsw)) + #endif diff --git a/arch/parisc/include/asm/spinlock.h b/arch/parisc/include/asm/spinlock.h index 6f84b6acc86e..8a63515f03bf 100644 --- a/arch/parisc/include/asm/spinlock.h +++ b/arch/parisc/include/asm/spinlock.h @@ -20,7 +20,6 @@ static inline void arch_spin_lock_flags(arch_spinlock_t *x, { volatile unsigned int *a; - mb(); a = __ldcw_align(x); while (__ldcw(a) == 0) while (*a == 0) @@ -30,17 +29,16 @@ static inline void arch_spin_lock_flags(arch_spinlock_t *x, local_irq_disable(); } else cpu_relax(); - mb(); } #define arch_spin_lock_flags arch_spin_lock_flags static inline void arch_spin_unlock(arch_spinlock_t *x) { volatile unsigned int *a; - mb(); + a = __ldcw_align(x); - *a = 1; mb(); + *a = 1; } static inline int arch_spin_trylock(arch_spinlock_t *x) @@ -48,10 +46,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *x) volatile unsigned int *a; int ret; - mb(); a = __ldcw_align(x); ret = __ldcw(a) != 0; - mb(); return ret; } diff --git a/arch/parisc/include/asm/unwind.h b/arch/parisc/include/asm/unwind.h index c73a3ee20226..f133b7efbebb 100644 --- a/arch/parisc/include/asm/unwind.h +++ b/arch/parisc/include/asm/unwind.h @@ -4,6 +4,9 @@ #include <linux/list.h> +/* Max number of levels to backtrace */ +#define MAX_UNWIND_ENTRIES 30 + /* From ABI specifications */ struct unwind_table_entry { unsigned int region_start; diff --git a/arch/parisc/include/uapi/asm/errno.h b/arch/parisc/include/uapi/asm/errno.h index fc0df353ff0d..87245c584784 100644 --- a/arch/parisc/include/uapi/asm/errno.h +++ b/arch/parisc/include/uapi/asm/errno.h @@ -113,7 +113,6 @@ #define ELOOP 249 /* Too many symbolic links encountered */ #define ENOSYS 251 /* Function not implemented */ -#define ENOTSUP 252 /* Function not implemented (POSIX.4 / HPUX) */ #define ECANCELLED 253 /* aio request was canceled before complete (POSIX.4 / HPUX) */ #define ECANCELED ECANCELLED /* SuSv3 and Solaris wants one 'L' */ diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index e95207c0565e..c7508f5717fb 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -482,6 +482,8 @@ .macro tlb_unlock0 spc,tmp #ifdef CONFIG_SMP or,COND(=) %r0,\spc,%r0 + sync + or,COND(=) %r0,\spc,%r0 stw \spc,0(\tmp) #endif .endm @@ -764,7 +766,6 @@ END(fault_vector_11) #endif /* Fault vector is separately protected and *must* be on its own page */ .align PAGE_SIZE -ENTRY(end_fault_vector) .import handle_interruption,code .import do_cpu_irq_mask,code @@ -776,7 +777,6 @@ ENTRY(end_fault_vector) */ ENTRY_CFI(ret_from_kernel_thread) - /* Call schedule_tail first though */ BL schedule_tail, %r2 nop @@ -815,8 +815,9 @@ ENTRY_CFI(_switch_to) LDREG TASK_THREAD_INFO(%r25), %r25 bv %r0(%r2) mtctl %r25,%cr30 +ENDPROC_CFI(_switch_to) -_switch_to_ret: +ENTRY_CFI(_switch_to_ret) mtctl %r0, %cr0 /* Needed for single stepping */ callee_rest callee_rest_float @@ -824,7 +825,7 @@ _switch_to_ret: LDREG -RP_OFFSET(%r30), %r2 bv %r0(%r2) copy %r26, %r28 -ENDPROC_CFI(_switch_to) +ENDPROC_CFI(_switch_to_ret) /* * Common rfi return path for interruptions, kernel execve, and @@ -885,12 +886,14 @@ ENTRY_CFI(syscall_exit_rfi) STREG %r19,PT_SR5(%r16) STREG %r19,PT_SR6(%r16) STREG %r19,PT_SR7(%r16) +ENDPROC_CFI(syscall_exit_rfi) -intr_return: +ENTRY_CFI(intr_return) /* check for reschedule */ mfctl %cr30,%r1 LDREG TI_FLAGS(%r1),%r19 /* sched.h: TIF_NEED_RESCHED */ bb,<,n %r19,31-TIF_NEED_RESCHED,intr_do_resched /* forward */ +ENDPROC_CFI(intr_return) .import do_notify_resume,code intr_check_sig: @@ -1046,7 +1049,6 @@ intr_extint: b do_cpu_irq_mask ldo R%intr_return(%r2), %r2 /* return to intr_return, not here */ -ENDPROC_CFI(syscall_exit_rfi) /* Generic interruptions (illegal insn, unaligned, page fault, etc) */ @@ -1997,12 +1999,9 @@ ENDPROC_CFI(syscall_exit) .align L1_CACHE_BYTES .globl mcount .type mcount, @function -ENTRY(mcount) +ENTRY_CFI(mcount, caller) _mcount: .export _mcount,data - .proc - .callinfo caller,frame=0 - .entry /* * The 64bit mcount() function pointer needs 4 dwords, of which the * first two are free. We optimize it here and put 2 instructions for @@ -2024,18 +2023,13 @@ ftrace_stub: .dword mcount .dword 0 /* code in head.S puts value of global gp here */ #endif - .exit - .procend -ENDPROC(mcount) +ENDPROC_CFI(mcount) #ifdef CONFIG_FUNCTION_GRAPH_TRACER .align 8 .globl return_to_handler .type return_to_handler, @function -ENTRY_CFI(return_to_handler) - .proc - .callinfo caller,frame=FRAME_SIZE - .entry +ENTRY_CFI(return_to_handler, caller,frame=FRAME_SIZE) .export parisc_return_to_handler,data parisc_return_to_handler: copy %r3,%r1 @@ -2074,8 +2068,6 @@ parisc_return_to_handler: bv %r0(%rp) #endif LDREGM -FRAME_SIZE(%sp),%r3 - .exit - .procend ENDPROC_CFI(return_to_handler) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ @@ -2085,31 +2077,30 @@ ENDPROC_CFI(return_to_handler) #ifdef CONFIG_IRQSTACKS /* void call_on_stack(unsigned long param1, void *func, unsigned long new_stack) */ -ENTRY_CFI(call_on_stack) +ENTRY_CFI(call_on_stack, FRAME=2*FRAME_SIZE,CALLS,SAVE_RP,SAVE_SP) copy %sp, %r1 /* Regarding the HPPA calling conventions for function pointers, we assume the PIC register is not changed across call. For CONFIG_64BIT, the argument pointer is left to point at the argument region allocated for the call to call_on_stack. */ + + /* Switch to new stack. We allocate two frames. */ + ldo 2*FRAME_SIZE(%arg2), %sp # ifdef CONFIG_64BIT - /* Switch to new stack. We allocate two 128 byte frames. */ - ldo 256(%arg2), %sp /* Save previous stack pointer and return pointer in frame marker */ - STREG %rp, -144(%sp) + STREG %rp, -FRAME_SIZE-RP_OFFSET(%sp) /* Calls always use function descriptor */ LDREG 16(%arg1), %arg1 bve,l (%arg1), %rp - STREG %r1, -136(%sp) - LDREG -144(%sp), %rp + STREG %r1, -FRAME_SIZE-REG_SZ(%sp) + LDREG -FRAME_SIZE-RP_OFFSET(%sp), %rp bve (%rp) - LDREG -136(%sp), %sp + LDREG -FRAME_SIZE-REG_SZ(%sp), %sp # else - /* Switch to new stack. We allocate two 64 byte frames. */ - ldo 128(%arg2), %sp /* Save previous stack pointer and return pointer in frame marker */ - STREG %r1, -68(%sp) - STREG %rp, -84(%sp) + STREG %r1, -FRAME_SIZE-REG_SZ(%sp) + STREG %rp, -FRAME_SIZE-RP_OFFSET(%sp) /* Calls use function descriptor if PLABEL bit is set */ bb,>=,n %arg1, 30, 1f depwi 0,31,2, %arg1 @@ -2117,9 +2108,9 @@ ENTRY_CFI(call_on_stack) 1: be,l 0(%sr4,%arg1), %sr0, %r31 copy %r31, %rp - LDREG -84(%sp), %rp + LDREG -FRAME_SIZE-RP_OFFSET(%sp), %rp bv (%rp) - LDREG -68(%sp), %sp + LDREG -FRAME_SIZE-REG_SZ(%sp), %sp # endif /* CONFIG_64BIT */ ENDPROC_CFI(call_on_stack) #endif /* CONFIG_IRQSTACKS */ diff --git a/arch/parisc/kernel/pacache.S b/arch/parisc/kernel/pacache.S index 22e6374ece44..f33bf2d306d6 100644 --- a/arch/parisc/kernel/pacache.S +++ b/arch/parisc/kernel/pacache.S @@ -44,10 +44,6 @@ .align 16 ENTRY_CFI(flush_tlb_all_local) - .proc - .callinfo NO_CALLS - .entry - /* * The pitlbe and pdtlbe instructions should only be used to * flush the entire tlb. Also, there needs to be no intervening @@ -189,18 +185,11 @@ fdtdone: 2: bv %r0(%r2) nop - - .exit - .procend ENDPROC_CFI(flush_tlb_all_local) .import cache_info,data ENTRY_CFI(flush_instruction_cache_local) - .proc - .callinfo NO_CALLS - .entry - load32 cache_info, %r1 /* Flush Instruction Cache */ @@ -256,18 +245,11 @@ fisync: mtsm %r22 /* restore I-bit */ bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_instruction_cache_local) .import cache_info, data ENTRY_CFI(flush_data_cache_local) - .proc - .callinfo NO_CALLS - .entry - load32 cache_info, %r1 /* Flush Data Cache */ @@ -324,9 +306,6 @@ fdsync: mtsm %r22 /* restore I-bit */ bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_data_cache_local) /* Macros to serialize TLB purge operations on SMP. */ @@ -353,6 +332,7 @@ ENDPROC_CFI(flush_data_cache_local) .macro tlb_unlock la,flags,tmp #ifdef CONFIG_SMP ldi 1,\tmp + sync stw \tmp,0(\la) mtsm \flags #endif @@ -361,10 +341,6 @@ ENDPROC_CFI(flush_data_cache_local) /* Clear page using kernel mapping. */ ENTRY_CFI(clear_page_asm) - .proc - .callinfo NO_CALLS - .entry - #ifdef CONFIG_64BIT /* Unroll the loop. */ @@ -423,18 +399,11 @@ ENTRY_CFI(clear_page_asm) #endif bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(clear_page_asm) /* Copy page using kernel mapping. */ ENTRY_CFI(copy_page_asm) - .proc - .callinfo NO_CALLS - .entry - #ifdef CONFIG_64BIT /* PA8x00 CPUs can consume 2 loads or 1 store per cycle. * Unroll the loop by hand and arrange insn appropriately. @@ -541,9 +510,6 @@ ENTRY_CFI(copy_page_asm) #endif bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(copy_page_asm) /* @@ -597,10 +563,6 @@ ENDPROC_CFI(copy_page_asm) */ ENTRY_CFI(copy_user_page_asm) - .proc - .callinfo NO_CALLS - .entry - /* Convert virtual `to' and `from' addresses to physical addresses. Move `from' physical address to non shadowed register. */ ldil L%(__PAGE_OFFSET), %r1 @@ -749,16 +711,9 @@ ENTRY_CFI(copy_user_page_asm) bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(copy_user_page_asm) ENTRY_CFI(clear_user_page_asm) - .proc - .callinfo NO_CALLS - .entry - tophys_r1 %r26 ldil L%(TMPALIAS_MAP_START), %r28 @@ -835,16 +790,9 @@ ENTRY_CFI(clear_user_page_asm) bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(clear_user_page_asm) ENTRY_CFI(flush_dcache_page_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%(TMPALIAS_MAP_START), %r28 #ifdef CONFIG_64BIT #if (TMPALIAS_MAP_START >= 0x80000000) @@ -902,16 +850,9 @@ ENTRY_CFI(flush_dcache_page_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_dcache_page_asm) ENTRY_CFI(flush_icache_page_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%(TMPALIAS_MAP_START), %r28 #ifdef CONFIG_64BIT #if (TMPALIAS_MAP_START >= 0x80000000) @@ -976,16 +917,9 @@ ENTRY_CFI(flush_icache_page_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_icache_page_asm) ENTRY_CFI(flush_kernel_dcache_page_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%dcache_stride, %r1 ldw R%dcache_stride(%r1), %r23 @@ -1019,16 +953,9 @@ ENTRY_CFI(flush_kernel_dcache_page_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_kernel_dcache_page_asm) ENTRY_CFI(purge_kernel_dcache_page_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%dcache_stride, %r1 ldw R%dcache_stride(%r1), %r23 @@ -1061,16 +988,9 @@ ENTRY_CFI(purge_kernel_dcache_page_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(purge_kernel_dcache_page_asm) ENTRY_CFI(flush_user_dcache_range_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%dcache_stride, %r1 ldw R%dcache_stride(%r1), %r23 ldo -1(%r23), %r21 @@ -1082,16 +1002,9 @@ ENTRY_CFI(flush_user_dcache_range_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_user_dcache_range_asm) ENTRY_CFI(flush_kernel_dcache_range_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%dcache_stride, %r1 ldw R%dcache_stride(%r1), %r23 ldo -1(%r23), %r21 @@ -1104,16 +1017,9 @@ ENTRY_CFI(flush_kernel_dcache_range_asm) syncdma bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_kernel_dcache_range_asm) ENTRY_CFI(purge_kernel_dcache_range_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%dcache_stride, %r1 ldw R%dcache_stride(%r1), %r23 ldo -1(%r23), %r21 @@ -1126,16 +1032,9 @@ ENTRY_CFI(purge_kernel_dcache_range_asm) syncdma bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(purge_kernel_dcache_range_asm) ENTRY_CFI(flush_user_icache_range_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%icache_stride, %r1 ldw R%icache_stride(%r1), %r23 ldo -1(%r23), %r21 @@ -1147,16 +1046,9 @@ ENTRY_CFI(flush_user_icache_range_asm) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_user_icache_range_asm) ENTRY_CFI(flush_kernel_icache_page) - .proc - .callinfo NO_CALLS - .entry - ldil L%icache_stride, %r1 ldw R%icache_stride(%r1), %r23 @@ -1190,16 +1082,9 @@ ENTRY_CFI(flush_kernel_icache_page) sync bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(flush_kernel_icache_page) ENTRY_CFI(flush_kernel_icache_range_asm) - .proc - .callinfo NO_CALLS - .entry - ldil L%icache_stride, %r1 ldw R%icache_stride(%r1), %r23 ldo -1(%r23), %r21 @@ -1211,8 +1096,6 @@ ENTRY_CFI(flush_kernel_icache_range_asm) sync bv %r0(%r2) nop - .exit - .procend ENDPROC_CFI(flush_kernel_icache_range_asm) __INIT @@ -1222,10 +1105,6 @@ ENDPROC_CFI(flush_kernel_icache_range_asm) */ .align 256 ENTRY_CFI(disable_sr_hashing_asm) - .proc - .callinfo NO_CALLS - .entry - /* * Switch to real mode */ @@ -1307,9 +1186,6 @@ srdis_done: 2: bv %r0(%r2) nop - .exit - - .procend ENDPROC_CFI(disable_sr_hashing_asm) .end diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index 6df07ce4f3c2..04c48f1ef3fb 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -21,13 +21,12 @@ #include <linux/init.h> #include <linux/gfp.h> #include <linux/mm.h> -#include <linux/pci.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/string.h> #include <linux/types.h> -#include <linux/scatterlist.h> -#include <linux/export.h> +#include <linux/dma-direct.h> +#include <linux/dma-noncoherent.h> #include <asm/cacheflush.h> #include <asm/dma.h> /* for DMA_CHUNK_SIZE */ @@ -395,7 +394,7 @@ pcxl_dma_init(void) __initcall(pcxl_dma_init); -static void *pa11_dma_alloc(struct device *dev, size_t size, +static void *pcxl_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { unsigned long vaddr; @@ -422,190 +421,60 @@ static void *pa11_dma_alloc(struct device *dev, size_t size, return (void *)vaddr; } -static void pa11_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle, unsigned long attrs) +static void *pcx_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { - int order; - - order = get_order(size); - size = 1 << (order + PAGE_SHIFT); - unmap_uncached_pages((unsigned long)vaddr, size); - pcxl_free_range((unsigned long)vaddr, size); - free_pages((unsigned long)__va(dma_handle), order); -} + void *addr; -static dma_addr_t pa11_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction direction, unsigned long attrs) -{ - void *addr = page_address(page) + offset; - BUG_ON(direction == DMA_NONE); + if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) + return NULL; - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - flush_kernel_dcache_range((unsigned long) addr, size); + addr = (void *)__get_free_pages(flag, get_order(size)); + if (addr) + *dma_handle = (dma_addr_t)virt_to_phys(addr); - return virt_to_phys(addr); + return addr; } -static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, - size_t size, enum dma_data_direction direction, - unsigned long attrs) +void *arch_dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - BUG_ON(direction == DMA_NONE); - - if (attrs & DMA_ATTR_SKIP_CPU_SYNC) - return; - if (direction == DMA_TO_DEVICE) - return; - - /* - * For PCI_DMA_FROMDEVICE this flush is not necessary for the - * simple map/unmap case. However, it IS necessary if if - * pci_dma_sync_single_* has been called and the buffer reused. - */ - - flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), size); + if (boot_cpu_data.cpu_type == pcxl2 || boot_cpu_data.cpu_type == pcxl) + return pcxl_dma_alloc(dev, size, dma_handle, gfp, attrs); + else + return pcx_dma_alloc(dev, size, dma_handle, gfp, attrs); } -static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction direction, - unsigned long attrs) +void arch_dma_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_handle, unsigned long attrs) { - int i; - struct scatterlist *sg; - - BUG_ON(direction == DMA_NONE); + int order = get_order(size); - for_each_sg(sglist, sg, nents, i) { - unsigned long vaddr = (unsigned long)sg_virt(sg); + if (boot_cpu_data.cpu_type == pcxl2 || boot_cpu_data.cpu_type == pcxl) { + size = 1 << (order + PAGE_SHIFT); + unmap_uncached_pages((unsigned long)vaddr, size); + pcxl_free_range((unsigned long)vaddr, size); - sg_dma_address(sg) = (dma_addr_t) virt_to_phys(vaddr); - sg_dma_len(sg) = sg->length; - - if (attrs & DMA_ATTR_SKIP_CPU_SYNC) - continue; - - flush_kernel_dcache_range(vaddr, sg->length); + vaddr = __va(dma_handle); } - return nents; -} - -static void pa11_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nents, enum dma_data_direction direction, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - BUG_ON(direction == DMA_NONE); - - if (attrs & DMA_ATTR_SKIP_CPU_SYNC) - return; - - if (direction == DMA_TO_DEVICE) - return; - - /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ - - for_each_sg(sglist, sg, nents, i) - flush_kernel_vmap_range(sg_virt(sg), sg->length); -} - -static void pa11_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); - - flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), - size); -} - -static void pa11_dma_sync_single_for_device(struct device *dev, - dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); - - flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), - size); + free_pages((unsigned long)vaddr, get_order(size)); } -static void pa11_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) +void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) { - int i; - struct scatterlist *sg; - - /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ - - for_each_sg(sglist, sg, nents, i) - flush_kernel_vmap_range(sg_virt(sg), sg->length); + flush_kernel_dcache_range((unsigned long)phys_to_virt(paddr), size); } -static void pa11_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) +void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir) { - int i; - struct scatterlist *sg; - - /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ - - for_each_sg(sglist, sg, nents, i) - flush_kernel_vmap_range(sg_virt(sg), sg->length); + flush_kernel_dcache_range((unsigned long)phys_to_virt(paddr), size); } -static void pa11_dma_cache_sync(struct device *dev, void *vaddr, size_t size, +void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction) { flush_kernel_dcache_range((unsigned long)vaddr, size); } - -const struct dma_map_ops pcxl_dma_ops = { - .alloc = pa11_dma_alloc, - .free = pa11_dma_free, - .map_page = pa11_dma_map_page, - .unmap_page = pa11_dma_unmap_page, - .map_sg = pa11_dma_map_sg, - .unmap_sg = pa11_dma_unmap_sg, - .sync_single_for_cpu = pa11_dma_sync_single_for_cpu, - .sync_single_for_device = pa11_dma_sync_single_for_device, - .sync_sg_for_cpu = pa11_dma_sync_sg_for_cpu, - .sync_sg_for_device = pa11_dma_sync_sg_for_device, - .cache_sync = pa11_dma_cache_sync, -}; - -static void *pcx_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) -{ - void *addr; - - if ((attrs & DMA_ATTR_NON_CONSISTENT) == 0) - return NULL; - - addr = (void *)__get_free_pages(flag, get_order(size)); - if (addr) - *dma_handle = (dma_addr_t)virt_to_phys(addr); - - return addr; -} - -static void pcx_dma_free(struct device *dev, size_t size, void *vaddr, - dma_addr_t iova, unsigned long attrs) -{ - free_pages((unsigned long)vaddr, get_order(size)); - return; -} - -const struct dma_map_ops pcx_dma_ops = { - .alloc = pcx_dma_alloc, - .free = pcx_dma_free, - .map_page = pa11_dma_map_page, - .unmap_page = pa11_dma_unmap_page, - .map_sg = pa11_dma_map_sg, - .unmap_sg = pa11_dma_unmap_sg, - .sync_single_for_cpu = pa11_dma_sync_single_for_cpu, - .sync_single_for_device = pa11_dma_sync_single_for_device, - .sync_sg_for_cpu = pa11_dma_sync_sg_for_cpu, - .sync_sg_for_device = pa11_dma_sync_sg_for_device, - .cache_sync = pa11_dma_cache_sync, -}; diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index b931745815e0..eb39e7e380d7 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -302,7 +302,7 @@ get_wchan(struct task_struct *p) ip = info.ip; if (!in_sched_functions(ip)) return ip; - } while (count++ < 16); + } while (count++ < MAX_UNWIND_ENTRIES); return 0; } diff --git a/arch/parisc/kernel/ptrace.c b/arch/parisc/kernel/ptrace.c index 7aa1d4d0d444..2582df1c529b 100644 --- a/arch/parisc/kernel/ptrace.c +++ b/arch/parisc/kernel/ptrace.c @@ -676,3 +676,103 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task) #endif return &user_parisc_native_view; } + + +/* HAVE_REGS_AND_STACK_ACCESS_API feature */ + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_INDEX(r,i) {.name = #r#i, .offset = offsetof(struct pt_regs, r[i])} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_INDEX(gr,0), + REG_OFFSET_INDEX(gr,1), + REG_OFFSET_INDEX(gr,2), + REG_OFFSET_INDEX(gr,3), + REG_OFFSET_INDEX(gr,4), + REG_OFFSET_INDEX(gr,5), + REG_OFFSET_INDEX(gr,6), + REG_OFFSET_INDEX(gr,7), + REG_OFFSET_INDEX(gr,8), + REG_OFFSET_INDEX(gr,9), + REG_OFFSET_INDEX(gr,10), + REG_OFFSET_INDEX(gr,11), + REG_OFFSET_INDEX(gr,12), + REG_OFFSET_INDEX(gr,13), + REG_OFFSET_INDEX(gr,14), + REG_OFFSET_INDEX(gr,15), + REG_OFFSET_INDEX(gr,16), + REG_OFFSET_INDEX(gr,17), + REG_OFFSET_INDEX(gr,18), + REG_OFFSET_INDEX(gr,19), + REG_OFFSET_INDEX(gr,20), + REG_OFFSET_INDEX(gr,21), + REG_OFFSET_INDEX(gr,22), + REG_OFFSET_INDEX(gr,23), + REG_OFFSET_INDEX(gr,24), + REG_OFFSET_INDEX(gr,25), + REG_OFFSET_INDEX(gr,26), + REG_OFFSET_INDEX(gr,27), + REG_OFFSET_INDEX(gr,28), + REG_OFFSET_INDEX(gr,29), + REG_OFFSET_INDEX(gr,30), + REG_OFFSET_INDEX(gr,31), + REG_OFFSET_INDEX(sr,0), + REG_OFFSET_INDEX(sr,1), + REG_OFFSET_INDEX(sr,2), + REG_OFFSET_INDEX(sr,3), + REG_OFFSET_INDEX(sr,4), + REG_OFFSET_INDEX(sr,5), + REG_OFFSET_INDEX(sr,6), + REG_OFFSET_INDEX(sr,7), + REG_OFFSET_INDEX(iasq,0), + REG_OFFSET_INDEX(iasq,1), + REG_OFFSET_INDEX(iaoq,0), + REG_OFFSET_INDEX(iaoq,1), + REG_OFFSET_NAME(cr27), + REG_OFFSET_NAME(ksp), + REG_OFFSET_NAME(kpc), + REG_OFFSET_NAME(sar), + REG_OFFSET_NAME(iir), + REG_OFFSET_NAME(isr), + REG_OFFSET_NAME(ior), + REG_OFFSET_NAME(ipsw), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_query_register_name() - query register name from its offset + * @offset: the offset of a register in struct pt_regs. + * + * regs_query_register_name() returns the name of a register from its + * offset in struct pt_regs. If the @offset is invalid, this returns NULL; + */ +const char *regs_query_register_name(unsigned int offset) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (roff->offset == offset) + return roff->name; + return NULL; +} diff --git a/arch/parisc/kernel/real2.S b/arch/parisc/kernel/real2.S index cc9963421a19..2b16d8d6598f 100644 --- a/arch/parisc/kernel/real2.S +++ b/arch/parisc/kernel/real2.S @@ -35,12 +35,6 @@ real32_stack: real64_stack: .block 8192 -#ifdef CONFIG_64BIT -# define REG_SZ 8 -#else -# define REG_SZ 4 -#endif - #define N_SAVED_REGS 9 save_cr_space: diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index 8d3a7b80ac42..4e87c35c22b7 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -97,14 +97,12 @@ void __init dma_ops_init(void) panic( "PA-RISC Linux currently only supports machines that conform to\n" "the PA-RISC 1.1 or 2.0 architecture specification.\n"); - case pcxs: - case pcxt: - hppa_dma_ops = &pcx_dma_ops; - break; case pcxl2: pa7300lc_init(); case pcxl: /* falls through */ - hppa_dma_ops = &pcxl_dma_ops; + case pcxs: + case pcxt: + hppa_dma_ops = &dma_noncoherent_ops; break; default: break; diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index e775f80ae28c..5f7e57fcaeef 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -629,11 +629,12 @@ cas_action: stw %r1, 4(%sr2,%r20) #endif /* The load and store could fail */ -1: ldw,ma 0(%r26), %r28 +1: ldw 0(%r26), %r28 sub,<> %r28, %r25, %r0 -2: stw,ma %r24, 0(%r26) +2: stw %r24, 0(%r26) /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) + sync + stw %r20, 0(%sr2,%r20) #if ENABLE_LWS_DEBUG /* Clear thread register indicator */ stw %r0, 4(%sr2,%r20) @@ -647,6 +648,7 @@ cas_action: 3: /* Error occurred on load or store */ /* Free lock */ + sync stw %r20, 0(%sr2,%r20) #if ENABLE_LWS_DEBUG stw %r0, 4(%sr2,%r20) @@ -796,30 +798,30 @@ cas2_action: ldo 1(%r0),%r28 /* 8bit CAS */ -13: ldb,ma 0(%r26), %r29 +13: ldb 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -14: stb,ma %r24, 0(%r26) +14: stb %r24, 0(%r26) b cas2_end copy %r0, %r28 nop nop /* 16bit CAS */ -15: ldh,ma 0(%r26), %r29 +15: ldh 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -16: sth,ma %r24, 0(%r26) +16: sth %r24, 0(%r26) b cas2_end copy %r0, %r28 nop nop /* 32bit CAS */ -17: ldw,ma 0(%r26), %r29 +17: ldw 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -18: stw,ma %r24, 0(%r26) +18: stw %r24, 0(%r26) b cas2_end copy %r0, %r28 nop @@ -827,10 +829,10 @@ cas2_action: /* 64bit CAS */ #ifdef CONFIG_64BIT -19: ldd,ma 0(%r26), %r29 +19: ldd 0(%r26), %r29 sub,*= %r29, %r25, %r0 b,n cas2_end -20: std,ma %r24, 0(%r26) +20: std %r24, 0(%r26) copy %r0, %r28 #else /* Compare first word */ @@ -848,7 +850,8 @@ cas2_action: cas2_end: /* Free lock */ - stw,ma %r20, 0(%sr2,%r20) + sync + stw %r20, 0(%sr2,%r20) /* Enable interrupts */ ssm PSW_SM_I, %r0 /* Return to userspace, set no error */ @@ -858,6 +861,7 @@ cas2_end: 22: /* Error occurred on load or store */ /* Free lock */ + sync stw %r20, 0(%sr2,%r20) ssm PSW_SM_I, %r0 ldo 1(%r0),%r28 diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 4309ad31a874..318815212518 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -172,7 +172,7 @@ static void do_show_stack(struct unwind_frame_info *info) int i = 1; printk(KERN_CRIT "Backtrace:\n"); - while (i <= 16) { + while (i <= MAX_UNWIND_ENTRIES) { if (unwind_once(info) < 0 || info->ip == 0) break; diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c index 2ef83d78eec4..5cdf13069dd9 100644 --- a/arch/parisc/kernel/unwind.c +++ b/arch/parisc/kernel/unwind.c @@ -13,7 +13,6 @@ #include <linux/init.h> #include <linux/sched.h> #include <linux/slab.h> -#include <linux/kallsyms.h> #include <linux/sort.h> #include <linux/uaccess.h> @@ -117,7 +116,8 @@ unwind_table_init(struct unwind_table *table, const char *name, for (; start <= end; start++) { if (start < end && start->region_end > (start+1)->region_start) { - printk("WARNING: Out of order unwind entry! %p and %p\n", start, start+1); + pr_warn("Out of order unwind entry! %px and %px\n", + start, start+1); } start->region_start += base_addr; @@ -203,25 +203,60 @@ int __init unwind_init(void) return 0; } -#ifdef CONFIG_64BIT -#define get_func_addr(fptr) fptr[2] -#else -#define get_func_addr(fptr) fptr[0] -#endif - static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size) { - extern void handle_interruption(int, struct pt_regs *); - static unsigned long *hi = (unsigned long *)&handle_interruption; - - if (pc == get_func_addr(hi)) { + /* + * We have to use void * instead of a function pointer, because + * function pointers aren't a pointer to the function on 64-bit. + * Make them const so the compiler knows they live in .text + */ + extern void * const handle_interruption; + extern void * const ret_from_kernel_thread; + extern void * const syscall_exit; + extern void * const intr_return; + extern void * const _switch_to_ret; +#ifdef CONFIG_IRQSTACKS + extern void * const call_on_stack; +#endif /* CONFIG_IRQSTACKS */ + + if (pc == (unsigned long) &handle_interruption) { struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN); dbg("Unwinding through handle_interruption()\n"); info->prev_sp = regs->gr[30]; info->prev_ip = regs->iaoq[0]; + return 1; + } + + if (pc == (unsigned long) &ret_from_kernel_thread || + pc == (unsigned long) &syscall_exit) { + info->prev_sp = info->prev_ip = 0; + return 1; + } + + if (pc == (unsigned long) &intr_return) { + struct pt_regs *regs; + + dbg("Found intr_return()\n"); + regs = (struct pt_regs *)(info->sp - PT_SZ_ALGN); + info->prev_sp = regs->gr[30]; + info->prev_ip = regs->iaoq[0]; + info->rp = regs->gr[2]; + return 1; + } + + if (pc == (unsigned long) &_switch_to_ret) { + info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE; + info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET); + return 1; + } +#ifdef CONFIG_IRQSTACKS + if (pc == (unsigned long) &call_on_stack) { + info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ); + info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET); return 1; } +#endif return 0; } @@ -238,34 +273,8 @@ static void unwind_frame_regs(struct unwind_frame_info *info) if (e == NULL) { unsigned long sp; - dbg("Cannot find unwind entry for 0x%lx; forced unwinding\n", info->ip); - -#ifdef CONFIG_KALLSYMS - /* Handle some frequent special cases.... */ - { - char symname[KSYM_NAME_LEN]; - char *modname; - - kallsyms_lookup(info->ip, NULL, NULL, &modname, - symname); - - dbg("info->ip = 0x%lx, name = %s\n", info->ip, symname); - - if (strcmp(symname, "_switch_to_ret") == 0) { - info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE; - info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET); - dbg("_switch_to_ret @ %lx - setting " - "prev_sp=%lx prev_ip=%lx\n", - info->ip, info->prev_sp, - info->prev_ip); - return; - } else if (strcmp(symname, "ret_from_kernel_thread") == 0 || - strcmp(symname, "syscall_exit") == 0) { - info->prev_ip = info->prev_sp = 0; - return; - } - } -#endif + dbg("Cannot find unwind entry for %pS; forced unwinding\n", + (void *) info->ip); /* Since we are doing the unwinding blind, we don't know if we are adjusting the stack correctly or extracting the rp @@ -439,8 +448,8 @@ unsigned long return_address(unsigned int level) /* initialize unwind info */ asm volatile ("copy %%r30, %0" : "=r"(sp)); memset(&r, 0, sizeof(struct pt_regs)); - r.iaoq[0] = (unsigned long) current_text_addr(); - r.gr[2] = (unsigned long) __builtin_return_address(0); + r.iaoq[0] = _THIS_IP_; + r.gr[2] = _RET_IP_; r.gr[30] = sp; unwind_frame_init(&info, current, &r); diff --git a/arch/parisc/lib/lusercopy.S b/arch/parisc/lib/lusercopy.S index d4fe19806d57..b53fb6fedf06 100644 --- a/arch/parisc/lib/lusercopy.S +++ b/arch/parisc/lib/lusercopy.S @@ -64,9 +64,6 @@ */ ENTRY_CFI(lclear_user) - .proc - .callinfo NO_CALLS - .entry comib,=,n 0,%r25,$lclu_done get_sr $lclu_loop: @@ -81,13 +78,9 @@ $lclu_done: ldo 1(%r25),%r25 ASM_EXCEPTIONTABLE_ENTRY(1b,2b) - - .exit ENDPROC_CFI(lclear_user) - .procend - /* * long lstrnlen_user(char *s, long n) * @@ -97,9 +90,6 @@ ENDPROC_CFI(lclear_user) */ ENTRY_CFI(lstrnlen_user) - .proc - .callinfo NO_CALLS - .entry comib,= 0,%r25,$lslen_nzero copy %r26,%r24 get_sr @@ -111,7 +101,6 @@ $lslen_loop: $lslen_done: bv %r0(%r2) sub %r26,%r24,%r28 - .exit $lslen_nzero: b $lslen_done @@ -125,9 +114,6 @@ $lslen_nzero: ENDPROC_CFI(lstrnlen_user) - .procend - - /* * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len) @@ -186,10 +172,6 @@ ENDPROC_CFI(lstrnlen_user) save_len = r31 ENTRY_CFI(pa_memcpy) - .proc - .callinfo NO_CALLS - .entry - /* Last destination address */ add dst,len,end @@ -439,9 +421,6 @@ ENTRY_CFI(pa_memcpy) b .Lcopy_done 10: stw,ma t1,4(dstspc,dst) ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done) - - .exit ENDPROC_CFI(pa_memcpy) - .procend .end diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c index 2607d2d33405..74842d28a7a1 100644 --- a/arch/parisc/mm/init.c +++ b/arch/parisc/mm/init.c @@ -19,7 +19,6 @@ #include <linux/gfp.h> #include <linux/delay.h> #include <linux/init.h> -#include <linux/pci.h> /* for hppa_dma_ops and pcxl_dma_ops */ #include <linux/initrd.h> #include <linux/swap.h> #include <linux/unistd.h> @@ -616,17 +615,13 @@ void __init mem_init(void) free_all_bootmem(); #ifdef CONFIG_PA11 - if (hppa_dma_ops == &pcxl_dma_ops) { + if (boot_cpu_data.cpu_type == pcxl2 || boot_cpu_data.cpu_type == pcxl) { pcxl_dma_start = (unsigned long)SET_MAP_OFFSET(MAP_START); parisc_vmalloc_start = SET_MAP_OFFSET(pcxl_dma_start + PCXL_DMA_MAP_SIZE); - } else { - pcxl_dma_start = 0; - parisc_vmalloc_start = SET_MAP_OFFSET(MAP_START); - } -#else - parisc_vmalloc_start = SET_MAP_OFFSET(MAP_START); + } else #endif + parisc_vmalloc_start = SET_MAP_OFFSET(MAP_START); mem_init_print_info(NULL); diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index 2ea575cb3401..fb96206de317 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -243,6 +243,7 @@ endif cpu-as-$(CONFIG_4xx) += -Wa,-m405 cpu-as-$(CONFIG_ALTIVEC) += $(call as-option,-Wa$(comma)-maltivec) cpu-as-$(CONFIG_E200) += -Wa,-me200 +cpu-as-$(CONFIG_E500) += -Wa,-me500 cpu-as-$(CONFIG_PPC_BOOK3S_64) += -Wa,-mpower4 cpu-as-$(CONFIG_PPC_E500MC) += $(call as-option,-Wa$(comma)-me500mc) diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 682b3e6a1e21..963abf8bf1c0 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -18,18 +18,11 @@ * a "bne-" instruction at the end, so an isync is enough as a acquire barrier * on the platform without lwsync. */ -#define __atomic_op_acquire(op, args...) \ -({ \ - typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ - __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory"); \ - __ret; \ -}) - -#define __atomic_op_release(op, args...) \ -({ \ - __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory"); \ - op##_relaxed(args); \ -}) +#define __atomic_acquire_fence() \ + __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") + +#define __atomic_release_fence() \ + __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") static __inline__ int atomic_read(const atomic_t *v) { @@ -129,8 +122,6 @@ ATOMIC_OPS(xor, xor) #undef ATOMIC_OP_RETURN_RELAXED #undef ATOMIC_OP -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - static __inline__ void atomic_inc(atomic_t *v) { int t; @@ -145,6 +136,7 @@ static __inline__ void atomic_inc(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic_inc atomic_inc static __inline__ int atomic_inc_return_relaxed(atomic_t *v) { @@ -163,16 +155,6 @@ static __inline__ int atomic_inc_return_relaxed(atomic_t *v) return t; } -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - static __inline__ void atomic_dec(atomic_t *v) { int t; @@ -187,6 +169,7 @@ static __inline__ void atomic_dec(atomic_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic_dec atomic_dec static __inline__ int atomic_dec_return_relaxed(atomic_t *v) { @@ -218,7 +201,7 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) /** - * __atomic_add_unless - add unless the number is a given value + * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -226,13 +209,13 @@ static __inline__ int atomic_dec_return_relaxed(atomic_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) +static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int t; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER -"1: lwarx %0,0,%1 # __atomic_add_unless\n\ +"1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ cmpw 0,%0,%3 \n\ beq 2f \n\ add %0,%2,%0 \n" @@ -248,6 +231,7 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return t; } +#define atomic_fetch_add_unless atomic_fetch_add_unless /** * atomic_inc_not_zero - increment unless the number is zero @@ -280,9 +264,6 @@ static __inline__ int atomic_inc_not_zero(atomic_t *v) } #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) -#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) -#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) - /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1, even if @@ -412,8 +393,6 @@ ATOMIC64_OPS(xor, xor) #undef ATOMIC64_OP_RETURN_RELAXED #undef ATOMIC64_OP -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) - static __inline__ void atomic64_inc(atomic64_t *v) { long t; @@ -427,6 +406,7 @@ static __inline__ void atomic64_inc(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic64_inc atomic64_inc static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v) { @@ -444,16 +424,6 @@ static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v) return t; } -/* - * atomic64_inc_and_test - increment and test - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) - static __inline__ void atomic64_dec(atomic64_t *v) { long t; @@ -467,6 +437,7 @@ static __inline__ void atomic64_dec(atomic64_t *v) : "r" (&v->counter) : "cc", "xer"); } +#define atomic64_dec atomic64_dec static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v) { @@ -487,9 +458,6 @@ static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v) #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) - /* * Atomically test *v and decrement if it is greater than 0. * The function returns the old value of *v minus 1. @@ -513,6 +481,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) return t; } +#define atomic64_dec_if_positive atomic64_dec_if_positive #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic64_cmpxchg_relaxed(v, o, n) \ @@ -524,7 +493,7 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) /** - * atomic64_add_unless - add unless the number is a given value + * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -532,13 +501,13 @@ static __inline__ long atomic64_dec_if_positive(atomic64_t *v) * Atomically adds @a to @v, so long as it was not @u. * Returns the old value of @v. */ -static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) +static __inline__ long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) { long t; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER -"1: ldarx %0,0,%1 # __atomic_add_unless\n\ +"1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ cmpd 0,%0,%3 \n\ beq 2f \n\ add %0,%2,%0 \n" @@ -551,8 +520,9 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) : "r" (&v->counter), "r" (a), "r" (u) : "cc", "memory"); - return t != u; + return t; } +#define atomic64_fetch_add_unless atomic64_fetch_add_unless /** * atomic_inc64_not_zero - increment unless the number is zero @@ -582,6 +552,7 @@ static __inline__ int atomic64_inc_not_zero(atomic64_t *v) return t1 != 0; } +#define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) #endif /* __powerpc64__ */ diff --git a/arch/powerpc/include/asm/hw_breakpoint.h b/arch/powerpc/include/asm/hw_breakpoint.h index 8e7b09703ca4..27d6e3c8fde9 100644 --- a/arch/powerpc/include/asm/hw_breakpoint.h +++ b/arch/powerpc/include/asm/hw_breakpoint.h @@ -52,6 +52,7 @@ struct arch_hw_breakpoint { #include <asm/reg.h> #include <asm/debug.h> +struct perf_event_attr; struct perf_event; struct pmu; struct perf_sample_data; @@ -60,8 +61,10 @@ struct perf_sample_data; extern int hw_breakpoint_slots(int type); extern int arch_bp_generic_fields(int type, int *gen_bp_type); -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); int arch_install_hw_breakpoint(struct perf_event *bp); diff --git a/arch/powerpc/include/asm/kprobes.h b/arch/powerpc/include/asm/kprobes.h index 9f3be5c8a4a3..785c464b6588 100644 --- a/arch/powerpc/include/asm/kprobes.h +++ b/arch/powerpc/include/asm/kprobes.h @@ -88,7 +88,6 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_saved_msr; - struct pt_regs jprobe_saved_regs; struct prev_kprobe prev_kprobe; }; @@ -103,17 +102,6 @@ extern int kprobe_exceptions_notify(struct notifier_block *self, extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); extern int kprobe_handler(struct pt_regs *regs); extern int kprobe_post_handler(struct pt_regs *regs); -#ifdef CONFIG_KPROBES_ON_FTRACE -extern int __is_active_jprobe(unsigned long addr); -extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb); -#else -static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb) -{ - return 0; -} -#endif #else static inline int kprobe_handler(struct pt_regs *regs) { return 0; } static inline int kprobe_post_handler(struct pt_regs *regs) { return 0; } diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 896efa559996..b2f89b621b15 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -35,9 +35,9 @@ extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm( extern struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm, unsigned long ua, unsigned long entries); extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, - unsigned long ua, unsigned long *hpa); + unsigned long ua, unsigned int pageshift, unsigned long *hpa); extern long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem, - unsigned long ua, unsigned long *hpa); + unsigned long ua, unsigned int pageshift, unsigned long *hpa); extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem); extern void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem); #endif @@ -143,24 +143,33 @@ static inline void mm_context_remove_copro(struct mm_struct *mm) { int c; - c = atomic_dec_if_positive(&mm->context.copros); - - /* Detect imbalance between add and remove */ - WARN_ON(c < 0); - /* - * Need to broadcast a global flush of the full mm before - * decrementing active_cpus count, as the next TLBI may be - * local and the nMMU and/or PSL need to be cleaned up. - * Should be rare enough so that it's acceptable. + * When removing the last copro, we need to broadcast a global + * flush of the full mm, as the next TLBI may be local and the + * nMMU and/or PSL need to be cleaned up. + * + * Both the 'copros' and 'active_cpus' counts are looked at in + * flush_all_mm() to determine the scope (local/global) of the + * TLBIs, so we need to flush first before decrementing + * 'copros'. If this API is used by several callers for the + * same context, it can lead to over-flushing. It's hopefully + * not common enough to be a problem. * * Skip on hash, as we don't know how to do the proper flush * for the time being. Invalidations will remain global if - * used on hash. + * used on hash. Note that we can't drop 'copros' either, as + * it could make some invalidations local with no flush + * in-between. */ - if (c == 0 && radix_enabled()) { + if (radix_enabled()) { flush_all_mm(mm); - dec_mm_active_cpus(mm); + + c = atomic_dec_if_positive(&mm->context.copros); + /* Detect imbalance between add and remove */ + WARN_ON(c < 0); + + if (c == 0) + dec_mm_active_cpus(mm); } } #else diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index 80547dad37da..fec8a6773119 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -119,11 +119,9 @@ void arch_unregister_hw_breakpoint(struct perf_event *bp) /* * Check for virtual address in kernel space. */ -int arch_check_bp_in_kernelspace(struct perf_event *bp) +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - - return is_kernel_addr(info->address); + return is_kernel_addr(hw->address); } int arch_bp_generic_fields(int type, int *gen_bp_type) @@ -141,30 +139,31 @@ int arch_bp_generic_fields(int type, int *gen_bp_type) /* * Validate the arch-specific HW Breakpoint register settings */ -int arch_validate_hwbkpt_settings(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { int ret = -EINVAL, length_max; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); if (!bp) return ret; - info->type = HW_BRK_TYPE_TRANSLATE; - if (bp->attr.bp_type & HW_BREAKPOINT_R) - info->type |= HW_BRK_TYPE_READ; - if (bp->attr.bp_type & HW_BREAKPOINT_W) - info->type |= HW_BRK_TYPE_WRITE; - if (info->type == HW_BRK_TYPE_TRANSLATE) + hw->type = HW_BRK_TYPE_TRANSLATE; + if (attr->bp_type & HW_BREAKPOINT_R) + hw->type |= HW_BRK_TYPE_READ; + if (attr->bp_type & HW_BREAKPOINT_W) + hw->type |= HW_BRK_TYPE_WRITE; + if (hw->type == HW_BRK_TYPE_TRANSLATE) /* must set alteast read or write */ return ret; - if (!(bp->attr.exclude_user)) - info->type |= HW_BRK_TYPE_USER; - if (!(bp->attr.exclude_kernel)) - info->type |= HW_BRK_TYPE_KERNEL; - if (!(bp->attr.exclude_hv)) - info->type |= HW_BRK_TYPE_HYP; - info->address = bp->attr.bp_addr; - info->len = bp->attr.bp_len; + if (!attr->exclude_user) + hw->type |= HW_BRK_TYPE_USER; + if (!attr->exclude_kernel) + hw->type |= HW_BRK_TYPE_KERNEL; + if (!attr->exclude_hv) + hw->type |= HW_BRK_TYPE_HYP; + hw->address = attr->bp_addr; + hw->len = attr->bp_len; /* * Since breakpoint length can be a maximum of HW_BREAKPOINT_LEN(8) @@ -178,12 +177,12 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) if (cpu_has_feature(CPU_FTR_DAWR)) { length_max = 512 ; /* 64 doublewords */ /* DAWR region can't cross 512 boundary */ - if ((bp->attr.bp_addr >> 9) != - ((bp->attr.bp_addr + bp->attr.bp_len - 1) >> 9)) + if ((attr->bp_addr >> 9) != + ((attr->bp_addr + attr->bp_len - 1) >> 9)) return -EINVAL; } - if (info->len > - (length_max - (info->address & HW_BREAKPOINT_ALIGN))) + if (hw->len > + (length_max - (hw->address & HW_BREAKPOINT_ALIGN))) return -EINVAL; return 0; } diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S index e734f6e45abc..689306118b48 100644 --- a/arch/powerpc/kernel/idle_book3s.S +++ b/arch/powerpc/kernel/idle_book3s.S @@ -144,7 +144,9 @@ power9_restore_additional_sprs: mtspr SPRN_MMCR1, r4 ld r3, STOP_MMCR2(r13) + ld r4, PACA_SPRG_VDSO(r13) mtspr SPRN_MMCR2, r3 + mtspr SPRN_SPRG3, r4 blr /* diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c index 7a1f99f1b47f..e4a49c051325 100644 --- a/arch/powerpc/kernel/kprobes-ftrace.c +++ b/arch/powerpc/kernel/kprobes-ftrace.c @@ -25,50 +25,6 @@ #include <linux/preempt.h> #include <linux/ftrace.h> -/* - * This is called from ftrace code after invoking registered handlers to - * disambiguate regs->nip changes done by jprobes and livepatch. We check if - * there is an active jprobe at the provided address (mcount location). - */ -int __is_active_jprobe(unsigned long addr) -{ - if (!preemptible()) { - struct kprobe *p = raw_cpu_read(current_kprobe); - return (p && (unsigned long)p->addr == addr) ? 1 : 0; - } - - return 0; -} - -static nokprobe_inline -int __skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb, unsigned long orig_nip) -{ - /* - * Emulate singlestep (and also recover regs->nip) - * as if there is a nop - */ - regs->nip = (unsigned long)p->addr + MCOUNT_INSN_SIZE; - if (unlikely(p->post_handler)) { - kcb->kprobe_status = KPROBE_HIT_SSDONE; - p->post_handler(p, regs, 0); - } - __this_cpu_write(current_kprobe, NULL); - if (orig_nip) - regs->nip = orig_nip; - return 1; -} - -int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb) -{ - if (kprobe_ftrace(p)) - return __skip_singlestep(p, regs, kcb, 0); - else - return 0; -} -NOKPROBE_SYMBOL(skip_singlestep); - /* Ftrace callback handler for kprobes */ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, struct ftrace_ops *ops, struct pt_regs *regs) @@ -76,18 +32,14 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, struct kprobe *p; struct kprobe_ctlblk *kcb; - preempt_disable(); - p = get_kprobe((kprobe_opcode_t *)nip); if (unlikely(!p) || kprobe_disabled(p)) - goto end; + return; kcb = get_kprobe_ctlblk(); if (kprobe_running()) { kprobes_inc_nmissed_count(p); } else { - unsigned long orig_nip = regs->nip; - /* * On powerpc, NIP is *before* this instruction for the * pre handler @@ -96,19 +48,23 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (!p->pre_handler || !p->pre_handler(p, regs)) - __skip_singlestep(p, regs, kcb, orig_nip); - else { + if (!p->pre_handler || !p->pre_handler(p, regs)) { /* - * If pre_handler returns !0, it sets regs->nip and - * resets current kprobe. In this case, we should not - * re-enable preemption. + * Emulate singlestep (and also recover regs->nip) + * as if there is a nop */ - return; + regs->nip += MCOUNT_INSN_SIZE; + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } } + /* + * If pre_handler returns !0, it changes regs->nip. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); } -end: - preempt_enable_no_resched(); } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index e4c5bf33970b..5c60bb0f927f 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -317,25 +317,17 @@ int kprobe_handler(struct pt_regs *regs) } prepare_singlestep(p, regs); return 1; - } else { - if (*addr != BREAKPOINT_INSTRUCTION) { - /* If trap variant, then it belongs not to us */ - kprobe_opcode_t cur_insn = *addr; - if (is_trap(cur_insn)) - goto no_kprobe; - /* The breakpoint instruction was removed by - * another cpu right after we hit, no further - * handling of this interrupt is appropriate - */ - ret = 1; + } else if (*addr != BREAKPOINT_INSTRUCTION) { + /* If trap variant, then it belongs not to us */ + kprobe_opcode_t cur_insn = *addr; + + if (is_trap(cur_insn)) goto no_kprobe; - } - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - if (!skip_singlestep(p, regs, kcb)) - goto ss_probe; - ret = 1; - } + /* The breakpoint instruction was removed by + * another cpu right after we hit, no further + * handling of this interrupt is appropriate + */ + ret = 1; } goto no_kprobe; } @@ -350,7 +342,7 @@ int kprobe_handler(struct pt_regs *regs) */ kprobe_opcode_t cur_insn = *addr; if (is_trap(cur_insn)) - goto no_kprobe; + goto no_kprobe; /* * The breakpoint instruction was removed right * after we hit it. Another cpu has removed @@ -366,11 +358,13 @@ int kprobe_handler(struct pt_regs *regs) kcb->kprobe_status = KPROBE_HIT_ACTIVE; set_current_kprobe(p, regs, kcb); - if (p->pre_handler && p->pre_handler(p, regs)) - /* handler has already set things up, so skip ss setup */ + if (p->pre_handler && p->pre_handler(p, regs)) { + /* handler changed execution path, so skip ss setup */ + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; + } -ss_probe: if (p->ainsn.boostable >= 0) { ret = try_to_emulate(p, regs); @@ -611,60 +605,6 @@ unsigned long arch_deref_entry_point(void *entry) } NOKPROBE_SYMBOL(arch_deref_entry_point); -int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); - - /* setup return addr to the jprobe handler routine */ - regs->nip = arch_deref_entry_point(jp->entry); -#ifdef PPC64_ELF_ABI_v2 - regs->gpr[12] = (unsigned long)jp->entry; -#elif defined(PPC64_ELF_ABI_v1) - regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc); -#endif - - /* - * jprobes use jprobe_return() which skips the normal return - * path of the function, and this messes up the accounting of the - * function graph tracer. - * - * Pause function graph tracing while performing the jprobe function. - */ - pause_graph_tracing(); - - return 1; -} -NOKPROBE_SYMBOL(setjmp_pre_handler); - -void __used jprobe_return(void) -{ - asm volatile("jprobe_return_trap:\n" - "trap\n" - ::: "memory"); -} -NOKPROBE_SYMBOL(jprobe_return); - -int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - if (regs->nip != ppc_kallsyms_lookup_name("jprobe_return_trap")) { - pr_debug("longjmp_break_handler NIP (0x%lx) does not match jprobe_return_trap (0x%lx)\n", - regs->nip, ppc_kallsyms_lookup_name("jprobe_return_trap")); - return 0; - } - - memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); - /* It's OK to start function graph tracing again */ - unpause_graph_tracing(); - preempt_enable_no_resched(); - return 1; -} -NOKPROBE_SYMBOL(longjmp_break_handler); - static struct kprobe trampoline_p = { .addr = (kprobe_opcode_t *) &kretprobe_trampoline, .pre_handler = trampoline_probe_handler diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index fe9733ffffaa..471aac313b89 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -42,6 +42,8 @@ #include <asm/ppc-pci.h> #include <asm/eeh.h> +#include "../../../drivers/pci/pci.h" + /* hose_spinlock protects accesses to the the phb_bitmap. */ static DEFINE_SPINLOCK(hose_spinlock); LIST_HEAD(hose_list); @@ -1014,7 +1016,7 @@ void pcibios_setup_bus_devices(struct pci_bus *bus) /* Cardbus can call us to add new devices to a bus, so ignore * those who are already fully discovered */ - if (dev->is_added) + if (pci_dev_is_added(dev)) continue; pcibios_setup_device(dev); diff --git a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S index 9a5b5a513604..32476a6e4e9c 100644 --- a/arch/powerpc/kernel/trace/ftrace_64_mprofile.S +++ b/arch/powerpc/kernel/trace/ftrace_64_mprofile.S @@ -104,39 +104,13 @@ ftrace_regs_call: bl ftrace_stub nop - /* Load the possibly modified NIP */ - ld r15, _NIP(r1) - + /* Load ctr with the possibly modified NIP */ + ld r3, _NIP(r1) + mtctr r3 #ifdef CONFIG_LIVEPATCH - cmpd r14, r15 /* has NIP been altered? */ + cmpd r14, r3 /* has NIP been altered? */ #endif -#if defined(CONFIG_LIVEPATCH) && defined(CONFIG_KPROBES_ON_FTRACE) - /* NIP has not been altered, skip over further checks */ - beq 1f - - /* Check if there is an active jprobe on us */ - subi r3, r14, 4 - bl __is_active_jprobe - nop - - /* - * If r3 == 1, then this is a kprobe/jprobe. - * else, this is livepatched function. - * - * The conditional branch for livepatch_handler below will use the - * result of this comparison. For kprobe/jprobe, we just need to branch to - * the new NIP, not call livepatch_handler. The branch below is bne, so we - * want CR0[EQ] to be true if this is a kprobe/jprobe. Which means we want - * CR0[EQ] = (r3 == 1). - */ - cmpdi r3, 1 -1: -#endif - - /* Load CTR with the possibly modified NIP */ - mtctr r15 - /* Restore gprs */ REST_GPR(0,r1) REST_10GPRS(2,r1) @@ -154,10 +128,7 @@ ftrace_regs_call: addi r1, r1, SWITCH_FRAME_SIZE #ifdef CONFIG_LIVEPATCH - /* - * Based on the cmpd or cmpdi above, if the NIP was altered and we're - * not on a kprobe/jprobe, then handle livepatch. - */ + /* Based on the cmpd above, if the NIP was altered handle livepatch */ bne- livepatch_handler #endif diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index d066e37551ec..8c456fa691a5 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -449,7 +449,7 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, /* This only handles v2 IOMMU type, v1 is handled via ioctl() */ return H_TOO_HARD; - if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, &hpa))) + if (WARN_ON_ONCE(mm_iommu_ua_to_hpa(mem, ua, tbl->it_page_shift, &hpa))) return H_HARDWARE; if (mm_iommu_mapped_inc(mem)) diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index 925fc316a104..5b298f5a1a14 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -279,7 +279,8 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, if (!mem) return H_TOO_HARD; - if (WARN_ON_ONCE_RM(mm_iommu_ua_to_hpa_rm(mem, ua, &hpa))) + if (WARN_ON_ONCE_RM(mm_iommu_ua_to_hpa_rm(mem, ua, tbl->it_page_shift, + &hpa))) return H_HARDWARE; pua = (void *) vmalloc_to_phys(pua); @@ -469,7 +470,8 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu, mem = mm_iommu_lookup_rm(vcpu->kvm->mm, ua, IOMMU_PAGE_SIZE_4K); if (mem) - prereg = mm_iommu_ua_to_hpa_rm(mem, ua, &tces) == 0; + prereg = mm_iommu_ua_to_hpa_rm(mem, ua, + IOMMU_PAGE_SHIFT_4K, &tces) == 0; } if (!prereg) { diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index de686b340f4a..ee4a8854985e 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -216,7 +216,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) wqp = kvm_arch_vcpu_wq(vcpu); if (swq_has_sleeper(wqp)) { - swake_up(wqp); + swake_up_one(wqp); ++vcpu->stat.halt_wakeup; } @@ -3188,7 +3188,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) } } - prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE); + prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE); if (kvmppc_vcore_check_block(vc)) { finish_swait(&vc->wq, &wait); @@ -3311,7 +3311,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) kvmppc_start_thread(vcpu, vc); trace_kvm_guest_enter(vcpu); } else if (vc->vcore_state == VCORE_SLEEPING) { - swake_up(&vc->wq); + swake_up_one(&vc->wq); } } diff --git a/arch/powerpc/mm/mmu_context_iommu.c b/arch/powerpc/mm/mmu_context_iommu.c index abb43646927a..a4ca57612558 100644 --- a/arch/powerpc/mm/mmu_context_iommu.c +++ b/arch/powerpc/mm/mmu_context_iommu.c @@ -19,6 +19,7 @@ #include <linux/hugetlb.h> #include <linux/swap.h> #include <asm/mmu_context.h> +#include <asm/pte-walk.h> static DEFINE_MUTEX(mem_list_mutex); @@ -27,6 +28,7 @@ struct mm_iommu_table_group_mem_t { struct rcu_head rcu; unsigned long used; atomic64_t mapped; + unsigned int pageshift; u64 ua; /* userspace address */ u64 entries; /* number of entries in hpas[] */ u64 *hpas; /* vmalloc'ed */ @@ -125,6 +127,8 @@ long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries, { struct mm_iommu_table_group_mem_t *mem; long i, j, ret = 0, locked_entries = 0; + unsigned int pageshift; + unsigned long flags; struct page *page = NULL; mutex_lock(&mem_list_mutex); @@ -159,6 +163,12 @@ long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries, goto unlock_exit; } + /* + * For a starting point for a maximum page size calculation + * we use @ua and @entries natural alignment to allow IOMMU pages + * smaller than huge pages but still bigger than PAGE_SIZE. + */ + mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT)); mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0]))); if (!mem->hpas) { kfree(mem); @@ -199,6 +209,23 @@ long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries, } } populate: + pageshift = PAGE_SHIFT; + if (PageCompound(page)) { + pte_t *pte; + struct page *head = compound_head(page); + unsigned int compshift = compound_order(head); + + local_irq_save(flags); /* disables as well */ + pte = find_linux_pte(mm->pgd, ua, NULL, &pageshift); + local_irq_restore(flags); + + /* Double check it is still the same pinned page */ + if (pte && pte_page(*pte) == head && + pageshift == compshift) + pageshift = max_t(unsigned int, pageshift, + PAGE_SHIFT); + } + mem->pageshift = min(mem->pageshift, pageshift); mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT; } @@ -349,7 +376,7 @@ struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm, EXPORT_SYMBOL_GPL(mm_iommu_find); long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, - unsigned long ua, unsigned long *hpa) + unsigned long ua, unsigned int pageshift, unsigned long *hpa) { const long entry = (ua - mem->ua) >> PAGE_SHIFT; u64 *va = &mem->hpas[entry]; @@ -357,6 +384,9 @@ long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, if (entry >= mem->entries) return -EFAULT; + if (pageshift > mem->pageshift) + return -EFAULT; + *hpa = *va | (ua & ~PAGE_MASK); return 0; @@ -364,7 +394,7 @@ long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa); long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem, - unsigned long ua, unsigned long *hpa) + unsigned long ua, unsigned int pageshift, unsigned long *hpa) { const long entry = (ua - mem->ua) >> PAGE_SHIFT; void *va = &mem->hpas[entry]; @@ -373,6 +403,9 @@ long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem, if (entry >= mem->entries) return -EFAULT; + if (pageshift > mem->pageshift) + return -EFAULT; + pa = (void *) vmalloc_to_phys(va); if (!pa) return -EFAULT; diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 380cbf9a40d9..c0a9bcd28356 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -286,6 +286,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u64 imm64; u8 *func; u32 true_cond; + u32 tmp_idx; /* * addrs[] maps a BPF bytecode address into a real offset from @@ -637,11 +638,7 @@ emit_clear: case BPF_STX | BPF_XADD | BPF_W: /* Get EA into TMP_REG_1 */ PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); - /* error if EA is not word-aligned */ - PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x03); - PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + 12); - PPC_LI(b2p[BPF_REG_0], 0); - PPC_JMP(exit_addr); + tmp_idx = ctx->idx * 4; /* load value from memory into TMP_REG_2 */ PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); /* add value from src_reg into this */ @@ -649,32 +646,16 @@ emit_clear: /* store result back */ PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); /* we're done if this succeeded */ - PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); - /* otherwise, let's try once more */ - PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); - PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); - PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); - /* exit if the store was not successful */ - PPC_LI(b2p[BPF_REG_0], 0); - PPC_BCC(COND_NE, exit_addr); + PPC_BCC_SHORT(COND_NE, tmp_idx); break; /* *(u64 *)(dst + off) += src */ case BPF_STX | BPF_XADD | BPF_DW: PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); - /* error if EA is not doubleword-aligned */ - PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x07); - PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (3*4)); - PPC_LI(b2p[BPF_REG_0], 0); - PPC_JMP(exit_addr); - PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); - PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); - PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); - PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); + tmp_idx = ctx->idx * 4; PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); - PPC_LI(b2p[BPF_REG_0], 0); - PPC_BCC(COND_NE, exit_addr); + PPC_BCC_SHORT(COND_NE, tmp_idx); break; /* diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 3f66fcf8ad99..19d8ab49d1bd 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -1469,7 +1469,7 @@ static int collect_events(struct perf_event *group, int max_count, } /* - * Add a event to the PMU. + * Add an event to the PMU. * If all events are not already frozen, then we disable and * re-enable the PMU in order to get hw_perf_enable to do the * actual work of reconfiguring the PMU. @@ -1548,7 +1548,7 @@ nocheck: } /* - * Remove a event from the PMU. + * Remove an event from the PMU. */ static void power_pmu_del(struct perf_event *event, int ef_flags) { @@ -1742,7 +1742,7 @@ static int power_pmu_commit_txn(struct pmu *pmu) /* * Return 1 if we might be able to put event on a limited PMC, * or 0 if not. - * A event can only go on a limited PMC if it counts something + * An event can only go on a limited PMC if it counts something * that a limited PMC can count, doesn't require interrupts, and * doesn't exclude any processor mode. */ diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 5bd0eb6681bc..70b2e1e0f23c 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -46,6 +46,7 @@ #include "powernv.h" #include "pci.h" +#include "../../../../drivers/pci/pci.h" #define PNV_IODA1_M64_NUM 16 /* Number of M64 BARs */ #define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */ @@ -3138,7 +3139,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev) struct pci_dn *pdn; int mul, total_vfs; - if (!pdev->is_physfn || pdev->is_added) + if (!pdev->is_physfn || pci_dev_is_added(pdev)) return; pdn = pci_get_pdn(pdev); diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 139f0af6c3d9..8a4868a3964b 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -71,6 +71,7 @@ #include <asm/security_features.h> #include "pseries.h" +#include "../../../../drivers/pci/pci.h" int CMO_PrPSP = -1; int CMO_SecPSP = -1; @@ -664,7 +665,7 @@ static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev) const int *indexes; struct device_node *dn = pci_device_to_OF_node(pdev); - if (!pdev->is_physfn || pdev->is_added) + if (!pdev->is_physfn || pci_dev_is_added(pdev)) return; /*Firmware must support open sriov otherwise dont configure*/ indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 47166ad2a669..196978733e64 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -2734,7 +2734,7 @@ generic_inst_dump(unsigned long adr, long count, int praddr, { int nr, dotted; unsigned long first_adr; - unsigned long inst, last_inst = 0; + unsigned int inst, last_inst = 0; unsigned char val[4]; dotted = 0; @@ -2758,7 +2758,7 @@ generic_inst_dump(unsigned long adr, long count, int praddr, dotted = 0; last_inst = inst; if (praddr) - printf(REG" %.8lx", adr, inst); + printf(REG" %.8x", adr, inst); printf("\t"); dump_func(inst, adr); printf("\n"); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f12680c9b947..4764fdeb4f1f 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -107,6 +107,7 @@ config ARCH_RV32I select GENERIC_LIB_ASHLDI3 select GENERIC_LIB_ASHRDI3 select GENERIC_LIB_LSHRDI3 + select GENERIC_LIB_UCMPDI2 config ARCH_RV64I bool "RV64I" diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h index 855115ace98c..c452359c9cb8 100644 --- a/arch/riscv/include/asm/atomic.h +++ b/arch/riscv/include/asm/atomic.h @@ -25,18 +25,11 @@ #define ATOMIC_INIT(i) { (i) } -#define __atomic_op_acquire(op, args...) \ -({ \ - typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ - __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory"); \ - __ret; \ -}) - -#define __atomic_op_release(op, args...) \ -({ \ - __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); \ - op##_relaxed(args); \ -}) +#define __atomic_acquire_fence() \ + __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") + +#define __atomic_release_fence() \ + __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); static __always_inline int atomic_read(const atomic_t *v) { @@ -209,130 +202,8 @@ ATOMIC_OPS(xor, xor, i) #undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN -/* - * The extra atomic operations that are constructed from one of the core - * AMO-based operations above (aside from sub, which is easier to fit above). - * These are required to perform a full barrier, but they're OK this way - * because atomic_*_return is also required to perform a full barrier. - * - */ -#define ATOMIC_OP(op, func_op, comp_op, I, c_type, prefix) \ -static __always_inline \ -bool atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_##func_op##_return(i, v) comp_op I; \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS(op, func_op, comp_op, I) \ - ATOMIC_OP(op, func_op, comp_op, I, int, ) -#else -#define ATOMIC_OPS(op, func_op, comp_op, I) \ - ATOMIC_OP(op, func_op, comp_op, I, int, ) \ - ATOMIC_OP(op, func_op, comp_op, I, long, 64) -#endif - -ATOMIC_OPS(add_and_test, add, ==, 0) -ATOMIC_OPS(sub_and_test, sub, ==, 0) -ATOMIC_OPS(add_negative, add, <, 0) - -#undef ATOMIC_OP -#undef ATOMIC_OPS - -#define ATOMIC_OP(op, func_op, I, c_type, prefix) \ -static __always_inline \ -void atomic##prefix##_##op(atomic##prefix##_t *v) \ -{ \ - atomic##prefix##_##func_op(I, v); \ -} - -#define ATOMIC_FETCH_OP(op, func_op, I, c_type, prefix) \ -static __always_inline \ -c_type atomic##prefix##_fetch_##op##_relaxed(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##func_op##_relaxed(I, v); \ -} \ -static __always_inline \ -c_type atomic##prefix##_fetch_##op(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##func_op(I, v); \ -} - -#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, c_type, prefix) \ -static __always_inline \ -c_type atomic##prefix##_##op##_return_relaxed(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##op##_relaxed(v) c_op I; \ -} \ -static __always_inline \ -c_type atomic##prefix##_##op##_return(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_fetch_##op(v) c_op I; \ -} - -#ifdef CONFIG_GENERIC_ATOMIC64 -#define ATOMIC_OPS(op, asm_op, c_op, I) \ - ATOMIC_OP( op, asm_op, I, int, ) \ - ATOMIC_FETCH_OP( op, asm_op, I, int, ) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) -#else -#define ATOMIC_OPS(op, asm_op, c_op, I) \ - ATOMIC_OP( op, asm_op, I, int, ) \ - ATOMIC_FETCH_OP( op, asm_op, I, int, ) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, int, ) \ - ATOMIC_OP( op, asm_op, I, long, 64) \ - ATOMIC_FETCH_OP( op, asm_op, I, long, 64) \ - ATOMIC_OP_RETURN(op, asm_op, c_op, I, long, 64) -#endif - -ATOMIC_OPS(inc, add, +, 1) -ATOMIC_OPS(dec, add, +, -1) - -#define atomic_inc_return_relaxed atomic_inc_return_relaxed -#define atomic_dec_return_relaxed atomic_dec_return_relaxed -#define atomic_inc_return atomic_inc_return -#define atomic_dec_return atomic_dec_return - -#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed -#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed -#define atomic_fetch_inc atomic_fetch_inc -#define atomic_fetch_dec atomic_fetch_dec - -#ifndef CONFIG_GENERIC_ATOMIC64 -#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed -#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed -#define atomic64_inc_return atomic64_inc_return -#define atomic64_dec_return atomic64_dec_return - -#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed -#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed -#define atomic64_fetch_inc atomic64_fetch_inc -#define atomic64_fetch_dec atomic64_fetch_dec -#endif - -#undef ATOMIC_OPS -#undef ATOMIC_OP -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN - -#define ATOMIC_OP(op, func_op, comp_op, I, prefix) \ -static __always_inline \ -bool atomic##prefix##_##op(atomic##prefix##_t *v) \ -{ \ - return atomic##prefix##_##func_op##_return(v) comp_op I; \ -} - -ATOMIC_OP(inc_and_test, inc, ==, 0, ) -ATOMIC_OP(dec_and_test, dec, ==, 0, ) -#ifndef CONFIG_GENERIC_ATOMIC64 -ATOMIC_OP(inc_and_test, inc, ==, 0, 64) -ATOMIC_OP(dec_and_test, dec, ==, 0, 64) -#endif - -#undef ATOMIC_OP - /* This is required to provide a full barrier on success. */ -static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int prev, rc; @@ -349,9 +220,10 @@ static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) : "memory"); return prev; } +#define atomic_fetch_add_unless atomic_fetch_add_unless #ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u) +static __always_inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) { long prev, rc; @@ -368,27 +240,7 @@ static __always_inline long __atomic64_add_unless(atomic64_t *v, long a, long u) : "memory"); return prev; } - -static __always_inline int atomic64_add_unless(atomic64_t *v, long a, long u) -{ - return __atomic64_add_unless(v, a, u) != u; -} -#endif - -/* - * The extra atomic operations that are constructed from one of the core - * LR/SC-based operations above. - */ -static __always_inline int atomic_inc_not_zero(atomic_t *v) -{ - return __atomic_add_unless(v, 1, 0); -} - -#ifndef CONFIG_GENERIC_ATOMIC64 -static __always_inline long atomic64_inc_not_zero(atomic64_t *v) -{ - return atomic64_add_unless(v, 1, 0); -} +#define atomic64_fetch_add_unless atomic64_fetch_add_unless #endif /* diff --git a/arch/riscv/include/uapi/asm/elf.h b/arch/riscv/include/uapi/asm/elf.h index 5cae4c30cd8e..1e0dfc36aab9 100644 --- a/arch/riscv/include/uapi/asm/elf.h +++ b/arch/riscv/include/uapi/asm/elf.h @@ -21,8 +21,13 @@ typedef struct user_regs_struct elf_gregset_t; typedef union __riscv_fp_state elf_fpregset_t; -#define ELF_RISCV_R_SYM(r_info) ((r_info) >> 32) -#define ELF_RISCV_R_TYPE(r_info) ((r_info) & 0xffffffff) +#if __riscv_xlen == 64 +#define ELF_RISCV_R_SYM(r_info) ELF64_R_SYM(r_info) +#define ELF_RISCV_R_TYPE(r_info) ELF64_R_TYPE(r_info) +#else +#define ELF_RISCV_R_SYM(r_info) ELF32_R_SYM(r_info) +#define ELF_RISCV_R_TYPE(r_info) ELF32_R_TYPE(r_info) +#endif /* * RISC-V relocation types diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index b74cbfbce2d0..7bcdaed15703 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -16,10 +16,6 @@ #include <linux/irqchip.h> #include <linux/irqdomain.h> -#ifdef CONFIG_RISCV_INTC -#include <linux/irqchip/irq-riscv-intc.h> -#endif - void __init init_IRQ(void) { irqchip_init(); diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 1d5e9b934b8c..3303ed2cd419 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -37,7 +37,7 @@ static int apply_r_riscv_64_rela(struct module *me, u32 *location, Elf_Addr v) static int apply_r_riscv_branch_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; u32 imm12 = (offset & 0x1000) << (31 - 12); u32 imm11 = (offset & 0x800) >> (11 - 7); u32 imm10_5 = (offset & 0x7e0) << (30 - 10); @@ -50,7 +50,7 @@ static int apply_r_riscv_branch_rela(struct module *me, u32 *location, static int apply_r_riscv_jal_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; u32 imm20 = (offset & 0x100000) << (31 - 20); u32 imm19_12 = (offset & 0xff000); u32 imm11 = (offset & 0x800) << (20 - 11); @@ -63,7 +63,7 @@ static int apply_r_riscv_jal_rela(struct module *me, u32 *location, static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; u16 imm8 = (offset & 0x100) << (12 - 8); u16 imm7_6 = (offset & 0xc0) >> (6 - 5); u16 imm5 = (offset & 0x20) >> (5 - 2); @@ -78,7 +78,7 @@ static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; u16 imm11 = (offset & 0x800) << (12 - 11); u16 imm10 = (offset & 0x400) >> (10 - 8); u16 imm9_8 = (offset & 0x300) << (12 - 11); @@ -96,7 +96,7 @@ static int apply_r_riscv_rvc_jump_rela(struct module *me, u32 *location, static int apply_r_riscv_pcrel_hi20_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; s32 hi20; if (offset != (s32)offset) { @@ -178,7 +178,7 @@ static int apply_r_riscv_lo12_s_rela(struct module *me, u32 *location, static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; s32 hi20; /* Always emit the got entry */ @@ -200,7 +200,7 @@ static int apply_r_riscv_got_hi20_rela(struct module *me, u32 *location, static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; s32 fill_v = offset; u32 hi20, lo12; @@ -227,7 +227,7 @@ static int apply_r_riscv_call_plt_rela(struct module *me, u32 *location, static int apply_r_riscv_call_rela(struct module *me, u32 *location, Elf_Addr v) { - s64 offset = (void *)v - (void *)location; + ptrdiff_t offset = (void *)v - (void *)location; s32 fill_v = offset; u32 hi20, lo12; @@ -263,14 +263,14 @@ static int apply_r_riscv_align_rela(struct module *me, u32 *location, static int apply_r_riscv_add32_rela(struct module *me, u32 *location, Elf_Addr v) { - *(u32 *)location += (*(u32 *)v); + *(u32 *)location += (u32)v; return 0; } static int apply_r_riscv_sub32_rela(struct module *me, u32 *location, Elf_Addr v) { - *(u32 *)location -= (*(u32 *)v); + *(u32 *)location -= (u32)v; return 0; } @@ -347,7 +347,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int j; for (j = 0; j < sechdrs[relsec].sh_size / sizeof(*rel); j++) { - u64 hi20_loc = + unsigned long hi20_loc = sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[j].r_offset; u32 hi20_type = ELF_RISCV_R_TYPE(rel[j].r_info); @@ -360,12 +360,12 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, Elf_Sym *hi20_sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_RISCV_R_SYM(rel[j].r_info); - u64 hi20_sym_val = + unsigned long hi20_sym_val = hi20_sym->st_value + rel[j].r_addend; /* Calculate lo12 */ - u64 offset = hi20_sym_val - hi20_loc; + size_t offset = hi20_sym_val - hi20_loc; if (IS_ENABLED(CONFIG_MODULE_SECTIONS) && hi20_type == R_RISCV_GOT_HI20) { offset = module_emit_got_entry( diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index ba3e80712797..9f82a7e34c64 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -50,7 +50,7 @@ static int riscv_gpr_set(struct task_struct *target, struct pt_regs *regs; regs = task_pt_regs(target); - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, ®s, 0, -1); + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, regs, 0, -1); return ret; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index ee44a48faf79..f0d2070866d4 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -220,8 +220,3 @@ void __init setup_arch(char **cmdline_p) riscv_fill_hwcap(); } -static int __init riscv_device_init(void) -{ - return of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); -} -subsys_initcall_sync(riscv_device_init); diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index c77df8142be2..58a522f9bcc3 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -28,7 +28,9 @@ static void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, }; +#ifdef CONFIG_ZONE_DMA32 max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, max_low_pfn)); +#endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; free_area_init_nodes(max_zone_pfns); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index baed39772c84..515240576930 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -106,7 +106,6 @@ config S390 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_WANTS_DYNAMIC_TASK_STRUCT - select ARCH_WANTS_UBSAN_NO_NULL select ARCH_WANT_IPC_PARSE_VERSION select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS2 @@ -140,12 +139,13 @@ config S390 select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER select HAVE_FUTEX_CMPXCHG if FUTEX - select HAVE_GCC_PLUGINS + select HAVE_GCC_PLUGINS if BROKEN select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZ4 select HAVE_KERNEL_LZMA select HAVE_KERNEL_LZO + select HAVE_KERNEL_UNCOMPRESSED select HAVE_KERNEL_XZ select HAVE_KPROBES select HAVE_KRETPROBES @@ -160,6 +160,7 @@ config S390 select HAVE_OPROFILE select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API + select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select MODULES_USE_ELF_RELA diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 68a690442be0..eee6703093c3 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -14,8 +14,18 @@ LD_BFD := elf64-s390 LDFLAGS := -m elf64_s390 KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC -KBUILD_CFLAGS += -m64 KBUILD_AFLAGS += -m64 +KBUILD_CFLAGS += -m64 +aflags_dwarf := -Wa,-gdwarf-2 +KBUILD_AFLAGS_DECOMPRESSOR := -m64 -D__ASSEMBLY__ +KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf)) +KBUILD_CFLAGS_DECOMPRESSOR := -m64 -O2 +KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY +KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float +KBUILD_CFLAGS_DECOMPRESSOR += -fno-asynchronous-unwind-tables +KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-option,-ffreestanding) +KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g) +KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,)) UTS_MACHINE := s390x STACK_SIZE := 16384 CHECKFLAGS += -D__s390__ -D__s390x__ @@ -52,18 +62,14 @@ cflags-y += -Wa,-I$(srctree)/arch/$(ARCH)/include # cflags-$(CONFIG_FRAME_POINTER) += -fno-optimize-sibling-calls -# old style option for packed stacks -ifeq ($(call cc-option-yn,-mkernel-backchain),y) -cflags-$(CONFIG_PACK_STACK) += -mkernel-backchain -D__PACK_STACK -aflags-$(CONFIG_PACK_STACK) += -D__PACK_STACK -endif - -# new style option for packed stacks ifeq ($(call cc-option-yn,-mpacked-stack),y) cflags-$(CONFIG_PACK_STACK) += -mpacked-stack -D__PACK_STACK aflags-$(CONFIG_PACK_STACK) += -D__PACK_STACK endif +KBUILD_AFLAGS_DECOMPRESSOR += $(aflags-y) +KBUILD_CFLAGS_DECOMPRESSOR += $(cflags-y) + ifeq ($(call cc-option-yn,-mstack-size=8192 -mstack-guard=128),y) cflags-$(CONFIG_CHECK_STACK) += -mstack-size=$(STACK_SIZE) ifneq ($(call cc-option-yn,-mstack-size=8192),y) @@ -71,8 +77,11 @@ cflags-$(CONFIG_CHECK_STACK) += -mstack-guard=$(CONFIG_STACK_GUARD) endif endif -ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y) -cflags-$(CONFIG_WARN_DYNAMIC_STACK) += -mwarn-dynamicstack +ifdef CONFIG_WARN_DYNAMIC_STACK + ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y) + KBUILD_CFLAGS += -mwarn-dynamicstack + KBUILD_CFLAGS_DECOMPRESSOR += -mwarn-dynamicstack + endif endif ifdef CONFIG_EXPOLINE @@ -82,6 +91,7 @@ ifdef CONFIG_EXPOLINE CC_FLAGS_EXPOLINE += -mindirect-branch-table export CC_FLAGS_EXPOLINE cflags-y += $(CC_FLAGS_EXPOLINE) -DCC_USING_EXPOLINE + aflags-y += -DCC_USING_EXPOLINE endif endif @@ -102,11 +112,12 @@ KBUILD_CFLAGS += -mbackchain -msoft-float $(cflags-y) KBUILD_CFLAGS += -pipe -fno-strength-reduce -Wno-sign-compare KBUILD_CFLAGS += -fno-asynchronous-unwind-tables $(cfi) KBUILD_AFLAGS += $(aflags-y) $(cfi) +export KBUILD_AFLAGS_DECOMPRESSOR +export KBUILD_CFLAGS_DECOMPRESSOR OBJCOPYFLAGS := -O binary -head-y := arch/s390/kernel/head.o -head-y += arch/s390/kernel/head64.o +head-y := arch/s390/kernel/head64.o # See arch/s390/Kbuild for content of core part of the kernel core-y += arch/s390/ @@ -121,7 +132,7 @@ boot := arch/s390/boot syscalls := arch/s390/kernel/syscalls tools := arch/s390/tools -all: image bzImage +all: bzImage #KBUILD_IMAGE is necessary for packaging targets like rpm-pkg, deb-pkg... KBUILD_IMAGE := $(boot)/bzImage @@ -129,7 +140,7 @@ KBUILD_IMAGE := $(boot)/bzImage install: vmlinux $(Q)$(MAKE) $(build)=$(boot) $@ -image bzImage: vmlinux +bzImage: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ zfcpdump: @@ -152,8 +163,7 @@ archprepare: # Don't use tabs in echo arguments define archhelp - echo '* image - Kernel image for IPL ($(boot)/image)' - echo '* bzImage - Compressed kernel image for IPL ($(boot)/bzImage)' + echo '* bzImage - Kernel image for IPL ($(boot)/bzImage)' echo ' install - Install kernel using' echo ' (your) ~/bin/$(INSTALLKERNEL) or' echo ' (distribution) /sbin/$(INSTALLKERNEL) or' diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index ee6a9c387c87..9bf8489df6e6 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -206,35 +206,28 @@ static int appldata_timer_handler(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - unsigned int len; - char buf[2]; + int timer_active = appldata_timer_active; + int zero = 0; + int one = 1; + int rc; + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &timer_active, + .maxlen = sizeof(int), + .extra1 = &zero, + .extra2 = &one, + }; + + rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; - if (!*lenp || *ppos) { - *lenp = 0; - return 0; - } - if (!write) { - strncpy(buf, appldata_timer_active ? "1\n" : "0\n", - ARRAY_SIZE(buf)); - len = strnlen(buf, ARRAY_SIZE(buf)); - if (len > *lenp) - len = *lenp; - if (copy_to_user(buffer, buf, len)) - return -EFAULT; - goto out; - } - len = *lenp; - if (copy_from_user(buf, buffer, len > sizeof(buf) ? sizeof(buf) : len)) - return -EFAULT; spin_lock(&appldata_timer_lock); - if (buf[0] == '1') + if (timer_active) __appldata_vtimer_setup(APPLDATA_ADD_TIMER); - else if (buf[0] == '0') + else __appldata_vtimer_setup(APPLDATA_DEL_TIMER); spin_unlock(&appldata_timer_lock); -out: - *lenp = len; - *ppos += len; return 0; } @@ -248,37 +241,24 @@ static int appldata_interval_handler(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - unsigned int len; - int interval; - char buf[16]; + int interval = appldata_interval; + int one = 1; + int rc; + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &interval, + .maxlen = sizeof(int), + .extra1 = &one, + }; - if (!*lenp || *ppos) { - *lenp = 0; - return 0; - } - if (!write) { - len = sprintf(buf, "%i\n", appldata_interval); - if (len > *lenp) - len = *lenp; - if (copy_to_user(buffer, buf, len)) - return -EFAULT; - goto out; - } - len = *lenp; - if (copy_from_user(buf, buffer, len > sizeof(buf) ? sizeof(buf) : len)) - return -EFAULT; - interval = 0; - sscanf(buf, "%i", &interval); - if (interval <= 0) - return -EINVAL; + rc = proc_dointvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; spin_lock(&appldata_timer_lock); appldata_interval = interval; __appldata_vtimer_setup(APPLDATA_MOD_TIMER); spin_unlock(&appldata_timer_lock); -out: - *lenp = len; - *ppos += len; return 0; } @@ -293,10 +273,17 @@ appldata_generic_handler(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { struct appldata_ops *ops = NULL, *tmp_ops; - unsigned int len; - int rc, found; - char buf[2]; struct list_head *lh; + int rc, found; + int active; + int zero = 0; + int one = 1; + struct ctl_table ctl_entry = { + .data = &active, + .maxlen = sizeof(int), + .extra1 = &zero, + .extra2 = &one, + }; found = 0; mutex_lock(&appldata_ops_mutex); @@ -317,31 +304,15 @@ appldata_generic_handler(struct ctl_table *ctl, int write, } mutex_unlock(&appldata_ops_mutex); - if (!*lenp || *ppos) { - *lenp = 0; + active = ops->active; + rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) { module_put(ops->owner); - return 0; - } - if (!write) { - strncpy(buf, ops->active ? "1\n" : "0\n", ARRAY_SIZE(buf)); - len = strnlen(buf, ARRAY_SIZE(buf)); - if (len > *lenp) - len = *lenp; - if (copy_to_user(buffer, buf, len)) { - module_put(ops->owner); - return -EFAULT; - } - goto out; - } - len = *lenp; - if (copy_from_user(buf, buffer, - len > sizeof(buf) ? sizeof(buf) : len)) { - module_put(ops->owner); - return -EFAULT; + return rc; } mutex_lock(&appldata_ops_mutex); - if ((buf[0] == '1') && (ops->active == 0)) { + if (active && (ops->active == 0)) { // protect work queue callback if (!try_module_get(ops->owner)) { mutex_unlock(&appldata_ops_mutex); @@ -359,7 +330,7 @@ appldata_generic_handler(struct ctl_table *ctl, int write, module_put(ops->owner); } else ops->active = 1; - } else if ((buf[0] == '0') && (ops->active == 1)) { + } else if (!active && (ops->active == 1)) { ops->active = 0; rc = appldata_diag(ops->record_nr, APPLDATA_STOP_REC, (unsigned long) ops->data, ops->size, @@ -370,9 +341,6 @@ appldata_generic_handler(struct ctl_table *ctl, int write, module_put(ops->owner); } mutex_unlock(&appldata_ops_mutex); -out: - *lenp = len; - *ppos += len; module_put(ops->owner); return 0; } diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index d1fa37fcce83..9e6668ee93de 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -3,19 +3,52 @@ # Makefile for the linux s390-specific parts of the memory manager. # -targets := image -targets += bzImage -subdir- := compressed +KCOV_INSTRUMENT := n +GCOV_PROFILE := n +UBSAN_SANITIZE := n -$(obj)/image: vmlinux FORCE - $(call if_changed,objcopy) +KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) +KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) + +# +# Use -march=z900 for als.c to be able to print an error +# message if the kernel is started on a machine which is too old +# +ifneq ($(CC_FLAGS_MARCH),-march=z900) +AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) +AFLAGS_head.o += -march=z900 +AFLAGS_REMOVE_mem.o += $(CC_FLAGS_MARCH) +AFLAGS_mem.o += -march=z900 +CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) +CFLAGS_als.o += -march=z900 +CFLAGS_REMOVE_sclp_early_core.o += $(CC_FLAGS_MARCH) +CFLAGS_sclp_early_core.o += -march=z900 +endif + +CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char + +obj-y := head.o als.o ebcdic.o sclp_early_core.o mem.o +targets := bzImage startup.a $(obj-y) +subdir- := compressed + +OBJECTS := $(addprefix $(obj)/,$(obj-y)) $(obj)/bzImage: $(obj)/compressed/vmlinux FORCE $(call if_changed,objcopy) -$(obj)/compressed/vmlinux: FORCE +$(obj)/compressed/vmlinux: $(obj)/startup.a FORCE $(Q)$(MAKE) $(build)=$(obj)/compressed $@ +quiet_cmd_ar = AR $@ + cmd_ar = rm -f $@; $(AR) rcsTP$(KBUILD_ARFLAGS) $@ $(filter $(OBJECTS), $^) + +$(obj)/startup.a: $(OBJECTS) FORCE + $(call if_changed,ar) + install: $(CONFIGURE) $(obj)/bzImage sh -x $(srctree)/$(obj)/install.sh $(KERNELRELEASE) $(obj)/bzImage \ System.map "$(INSTALL_PATH)" + +chkbss := $(OBJECTS) +chkbss-target := $(obj)/startup.a +include $(srctree)/arch/s390/scripts/Makefile.chkbss diff --git a/arch/s390/kernel/als.c b/arch/s390/boot/als.c index d1892bf36cab..d592e0d90d9f 100644 --- a/arch/s390/kernel/als.c +++ b/arch/s390/boot/als.c @@ -3,12 +3,10 @@ * Copyright IBM Corp. 2016 */ #include <linux/kernel.h> -#include <linux/init.h> #include <asm/processor.h> #include <asm/facility.h> #include <asm/lowcore.h> #include <asm/sclp.h> -#include "entry.h" /* * The code within this file will be called very early. It may _not_ @@ -18,9 +16,9 @@ * For temporary objects the stack (16k) should be used. */ -static unsigned long als[] __initdata = { FACILITIES_ALS }; +static unsigned long als[] = { FACILITIES_ALS }; -static void __init u16_to_hex(char *str, u16 val) +static void u16_to_hex(char *str, u16 val) { int i, num; @@ -33,9 +31,9 @@ static void __init u16_to_hex(char *str, u16 val) *str = '\0'; } -static void __init print_machine_type(void) +static void print_machine_type(void) { - static char mach_str[80] __initdata = "Detected machine-type number: "; + static char mach_str[80] = "Detected machine-type number: "; char type_str[5]; struct cpuid id; @@ -46,7 +44,7 @@ static void __init print_machine_type(void) sclp_early_printk(mach_str); } -static void __init u16_to_decimal(char *str, u16 val) +static void u16_to_decimal(char *str, u16 val) { int div = 1; @@ -60,9 +58,9 @@ static void __init u16_to_decimal(char *str, u16 val) *str = '\0'; } -static void __init print_missing_facilities(void) +static void print_missing_facilities(void) { - static char als_str[80] __initdata = "Missing facilities: "; + static char als_str[80] = "Missing facilities: "; unsigned long val; char val_str[6]; int i, j, first; @@ -95,7 +93,7 @@ static void __init print_missing_facilities(void) sclp_early_printk("See Principles of Operations for facility bits\n"); } -static void __init facility_mismatch(void) +static void facility_mismatch(void) { sclp_early_printk("The Linux kernel requires more recent processor hardware\n"); print_machine_type(); @@ -103,7 +101,7 @@ static void __init facility_mismatch(void) disabled_wait(0x8badcccc); } -void __init verify_facilities(void) +void verify_facilities(void) { int i; diff --git a/arch/s390/boot/compressed/.gitignore b/arch/s390/boot/compressed/.gitignore index 2088cc140629..45aeb4f08752 100644 --- a/arch/s390/boot/compressed/.gitignore +++ b/arch/s390/boot/compressed/.gitignore @@ -1,4 +1,5 @@ sizes.h vmlinux vmlinux.lds +vmlinux.scr.lds vmlinux.bin.full diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index 5766f7b9b271..04609478d18b 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -6,39 +6,29 @@ # KCOV_INSTRUMENT := n +GCOV_PROFILE := n +UBSAN_SANITIZE := n +obj-y := $(if $(CONFIG_KERNEL_UNCOMPRESSED),,head.o misc.o) piggy.o targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 -targets += misc.o piggy.o sizes.h head.o - -KBUILD_CFLAGS := -m64 -D__KERNEL__ -O2 -KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY -KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -msoft-float -KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -KBUILD_CFLAGS += $(call cc-option,-mpacked-stack) -KBUILD_CFLAGS += $(call cc-option,-ffreestanding) +targets += vmlinux.scr.lds $(obj-y) $(if $(CONFIG_KERNEL_UNCOMPRESSED),,sizes.h) -GCOV_PROFILE := n -UBSAN_SANITIZE := n +KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) +KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) -OBJECTS := $(addprefix $(objtree)/arch/s390/kernel/, head.o ebcdic.o als.o) -OBJECTS += $(objtree)/drivers/s390/char/sclp_early_core.o -OBJECTS += $(obj)/head.o $(obj)/misc.o $(obj)/piggy.o +OBJECTS := $(addprefix $(obj)/,$(obj-y)) LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T -$(obj)/vmlinux: $(obj)/vmlinux.lds $(OBJECTS) +$(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS) $(call if_changed,ld) -TRIM_HEAD_SIZE := 0x11000 - -sed-sizes := -e 's/^\([0-9a-fA-F]*\) . \(__bss_start\|_end\)$$/\#define SZ\2 (0x\1 - $(TRIM_HEAD_SIZE))/p' +# extract required uncompressed vmlinux symbols and adjust them to reflect offsets inside vmlinux.bin +sed-sizes := -e 's/^\([0-9a-fA-F]*\) . \(__bss_start\|_end\)$$/\#define SZ\2 (0x\1 - 0x100000)/p' quiet_cmd_sizes = GEN $@ cmd_sizes = $(NM) $< | sed -n $(sed-sizes) > $@ -quiet_cmd_trim_head = TRIM $@ - cmd_trim_head = tail -c +$$(($(TRIM_HEAD_SIZE) + 1)) $< > $@ - $(obj)/sizes.h: vmlinux $(call if_changed,sizes) @@ -48,21 +38,18 @@ $(obj)/head.o: $(obj)/sizes.h CFLAGS_misc.o += -I$(objtree)/$(obj) $(obj)/misc.o: $(obj)/sizes.h -OBJCOPYFLAGS_vmlinux.bin.full := -R .comment -S -$(obj)/vmlinux.bin.full: vmlinux +OBJCOPYFLAGS_vmlinux.bin := -R .comment -S +$(obj)/vmlinux.bin: vmlinux $(call if_changed,objcopy) -$(obj)/vmlinux.bin: $(obj)/vmlinux.bin.full - $(call if_changed,trim_head) - vmlinux.bin.all-y := $(obj)/vmlinux.bin -suffix-$(CONFIG_KERNEL_GZIP) := gz -suffix-$(CONFIG_KERNEL_BZIP2) := bz2 -suffix-$(CONFIG_KERNEL_LZ4) := lz4 -suffix-$(CONFIG_KERNEL_LZMA) := lzma -suffix-$(CONFIG_KERNEL_LZO) := lzo -suffix-$(CONFIG_KERNEL_XZ) := xz +suffix-$(CONFIG_KERNEL_GZIP) := .gz +suffix-$(CONFIG_KERNEL_BZIP2) := .bz2 +suffix-$(CONFIG_KERNEL_LZ4) := .lz4 +suffix-$(CONFIG_KERNEL_LZMA) := .lzma +suffix-$(CONFIG_KERNEL_LZO) := .lzo +suffix-$(CONFIG_KERNEL_XZ) := .xz $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) $(call if_changed,gzip) @@ -78,5 +65,9 @@ $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) $(call if_changed,xzkern) LDFLAGS_piggy.o := -r --format binary --oformat $(LD_BFD) -T -$(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix-y) +$(obj)/piggy.o: $(obj)/vmlinux.scr.lds $(obj)/vmlinux.bin$(suffix-y) $(call if_changed,ld) + +chkbss := $(filter-out $(obj)/misc.o $(obj)/piggy.o,$(OBJECTS)) +chkbss-target := $(obj)/vmlinux.bin +include $(srctree)/arch/s390/scripts/Makefile.chkbss diff --git a/arch/s390/boot/compressed/head.S b/arch/s390/boot/compressed/head.S index 9f94eca0f467..df8dbbc17bcc 100644 --- a/arch/s390/boot/compressed/head.S +++ b/arch/s390/boot/compressed/head.S @@ -15,7 +15,7 @@ #include "sizes.h" __HEAD -ENTRY(startup_continue) +ENTRY(startup_decompressor) basr %r13,0 # get base .LPG1: # setup stack @@ -23,7 +23,7 @@ ENTRY(startup_continue) aghi %r15,-160 brasl %r14,decompress_kernel # Set up registers for memory mover. We move the decompressed image to - # 0x11000, where startup_continue of the decompressed image is supposed + # 0x100000, where startup_continue of the decompressed image is supposed # to be. lgr %r4,%r2 lg %r2,.Loffset-.LPG1(%r13) @@ -33,7 +33,7 @@ ENTRY(startup_continue) la %r1,0x200 mvc 0(mover_end-mover,%r1),mover-.LPG1(%r13) # When the memory mover is done we pass control to - # arch/s390/kernel/head64.S:startup_continue which lives at 0x11000 in + # arch/s390/kernel/head64.S:startup_continue which lives at 0x100000 in # the decompressed image. lgr %r6,%r2 br %r1 @@ -47,6 +47,6 @@ mover_end: .Lstack: .quad 0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)) .Loffset: - .quad 0x11000 + .quad 0x100000 .Lmvsize: .quad SZ__bss_start diff --git a/arch/s390/boot/compressed/misc.c b/arch/s390/boot/compressed/misc.c index 511b2cc9b91a..f66ad73c205b 100644 --- a/arch/s390/boot/compressed/misc.c +++ b/arch/s390/boot/compressed/misc.c @@ -71,43 +71,6 @@ static int puts(const char *s) return 0; } -void *memset(void *s, int c, size_t n) -{ - char *xs; - - xs = s; - while (n--) - *xs++ = c; - return s; -} - -void *memcpy(void *dest, const void *src, size_t n) -{ - const char *s = src; - char *d = dest; - - while (n--) - *d++ = *s++; - return dest; -} - -void *memmove(void *dest, const void *src, size_t n) -{ - const char *s = src; - char *d = dest; - - if (d <= s) { - while (n--) - *d++ = *s++; - } else { - d += n; - s += n; - while (n--) - *--d = *--s; - } - return dest; -} - static void error(char *x) { unsigned long long psw = 0x000a0000deadbeefULL; diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index d43c2db12d30..b16ac8b3c439 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -23,13 +23,10 @@ SECTIONS *(.text.*) _etext = . ; } - .rodata.compressed : { - *(.rodata.compressed) - } .rodata : { _rodata = . ; *(.rodata) /* read-only data */ - *(.rodata.*) + *(EXCLUDE_FILE (*piggy.o) .rodata.compressed) _erodata = . ; } .data : { @@ -38,6 +35,15 @@ SECTIONS *(.data.*) _edata = . ; } + startup_continue = 0x100000; +#ifdef CONFIG_KERNEL_UNCOMPRESSED + . = 0x100000; +#else + . = ALIGN(8); +#endif + .rodata.compressed : { + *(.rodata.compressed) + } . = ALIGN(256); .bss : { _bss = . ; @@ -54,5 +60,6 @@ SECTIONS *(.eh_frame) *(__ex_table) *(*__ksymtab*) + *(___kcrctab*) } } diff --git a/arch/s390/boot/compressed/vmlinux.scr b/arch/s390/boot/compressed/vmlinux.scr.lds.S index 42a242597f34..ff01d18c9222 100644 --- a/arch/s390/boot/compressed/vmlinux.scr +++ b/arch/s390/boot/compressed/vmlinux.scr.lds.S @@ -2,10 +2,14 @@ SECTIONS { .rodata.compressed : { +#ifndef CONFIG_KERNEL_UNCOMPRESSED input_len = .; LONG(input_data_end - input_data) input_data = .; +#endif *(.data) +#ifndef CONFIG_KERNEL_UNCOMPRESSED output_len = . - 4; input_data_end = .; +#endif } } diff --git a/arch/s390/boot/ebcdic.c b/arch/s390/boot/ebcdic.c new file mode 100644 index 000000000000..7391e7d36086 --- /dev/null +++ b/arch/s390/boot/ebcdic.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../kernel/ebcdic.c" diff --git a/arch/s390/kernel/head.S b/arch/s390/boot/head.S index 5c42f16a54c4..f721913b73f1 100644 --- a/arch/s390/kernel/head.S +++ b/arch/s390/boot/head.S @@ -272,14 +272,14 @@ iplstart: .org 0x10000 ENTRY(startup) j .Lep_startup_normal - .org 0x10008 + .org EP_OFFSET # # This is a list of s390 kernel entry points. At address 0x1000f the number of # valid entry points is stored. # # IMPORTANT: Do not change this table, it is s390 kernel ABI! # - .ascii "S390EP" + .ascii EP_STRING .byte 0x00,0x01 # # kdump startup-code at 0x10010, running in 64 bit absolute addressing mode @@ -310,10 +310,11 @@ ENTRY(startup_kdump) l %r15,.Lstack-.LPG0(%r13) ahi %r15,-STACK_FRAME_OVERHEAD brasl %r14,verify_facilities -# For uncompressed images, continue in -# arch/s390/kernel/head64.S. For compressed images, continue in -# arch/s390/boot/compressed/head.S. +#ifdef CONFIG_KERNEL_UNCOMPRESSED jg startup_continue +#else + jg startup_decompressor +#endif .Lstack: .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)) diff --git a/arch/s390/kernel/head_kdump.S b/arch/s390/boot/head_kdump.S index 174d6959bf5b..174d6959bf5b 100644 --- a/arch/s390/kernel/head_kdump.S +++ b/arch/s390/boot/head_kdump.S diff --git a/arch/s390/boot/mem.S b/arch/s390/boot/mem.S new file mode 100644 index 000000000000..b33463633f03 --- /dev/null +++ b/arch/s390/boot/mem.S @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include "../lib/mem.S" diff --git a/arch/s390/boot/sclp_early_core.c b/arch/s390/boot/sclp_early_core.c new file mode 100644 index 000000000000..5a19fd7020b5 --- /dev/null +++ b/arch/s390/boot/sclp_early_core.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../../../drivers/s390/char/sclp_early_core.c" diff --git a/arch/s390/hypfs/hypfs_diag.c b/arch/s390/hypfs/hypfs_diag.c index a2945b289a29..3452e18bb1ca 100644 --- a/arch/s390/hypfs/hypfs_diag.c +++ b/arch/s390/hypfs/hypfs_diag.c @@ -497,7 +497,7 @@ static int hypfs_create_cpu_files(struct dentry *cpus_dir, void *cpu_info) } diag224_idx2name(cpu_info__ctidx(diag204_info_type, cpu_info), buffer); rc = hypfs_create_str(cpu_dir, "type", buffer); - return PTR_RET(rc); + return PTR_ERR_OR_ZERO(rc); } static void *hypfs_create_lpar_files(struct dentry *systems_dir, void *part_hdr) @@ -544,7 +544,7 @@ static int hypfs_create_phys_cpu_files(struct dentry *cpus_dir, void *cpu_info) return PTR_ERR(rc); diag224_idx2name(phys_cpu__ctidx(diag204_info_type, cpu_info), buffer); rc = hypfs_create_str(cpu_dir, "type", buffer); - return PTR_RET(rc); + return PTR_ERR_OR_ZERO(rc); } static void *hypfs_create_phys_files(struct dentry *parent_dir, void *phys_hdr) diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index 06b513d192b9..c681329fdeec 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c @@ -36,7 +36,7 @@ struct hypfs_sb_info { kuid_t uid; /* uid used for files and dirs */ kgid_t gid; /* gid used for files and dirs */ struct dentry *update_file; /* file to trigger update */ - time_t last_update; /* last update time in secs since 1970 */ + time64_t last_update; /* last update, CLOCK_MONOTONIC time */ struct mutex lock; /* lock to protect update process */ }; @@ -52,7 +52,7 @@ static void hypfs_update_update(struct super_block *sb) struct hypfs_sb_info *sb_info = sb->s_fs_info; struct inode *inode = d_inode(sb_info->update_file); - sb_info->last_update = get_seconds(); + sb_info->last_update = ktime_get_seconds(); inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); } @@ -179,7 +179,7 @@ static ssize_t hypfs_write_iter(struct kiocb *iocb, struct iov_iter *from) * to restart data collection in this case. */ mutex_lock(&fs_info->lock); - if (fs_info->last_update == get_seconds()) { + if (fs_info->last_update == ktime_get_seconds()) { rc = -EBUSY; goto out; } diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index c1bedb4c8de0..046e044a48d0 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -47,6 +47,50 @@ struct ap_queue_status { }; /** + * ap_intructions_available() - Test if AP instructions are available. + * + * Returns 0 if the AP instructions are installed. + */ +static inline int ap_instructions_available(void) +{ + register unsigned long reg0 asm ("0") = AP_MKQID(0, 0); + register unsigned long reg1 asm ("1") = -ENODEV; + register unsigned long reg2 asm ("2"); + + asm volatile( + " .long 0xb2af0000\n" /* PQAP(TAPQ) */ + "0: la %0,0\n" + "1:\n" + EX_TABLE(0b, 1b) + : "+d" (reg1), "=d" (reg2) + : "d" (reg0) + : "cc"); + return reg1; +} + +/** + * ap_tapq(): Test adjunct processor queue. + * @qid: The AP queue number + * @info: Pointer to queue descriptor + * + * Returns AP queue status structure. + */ +static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info) +{ + register unsigned long reg0 asm ("0") = qid; + register struct ap_queue_status reg1 asm ("1"); + register unsigned long reg2 asm ("2"); + + asm volatile(".long 0xb2af0000" /* PQAP(TAPQ) */ + : "=d" (reg1), "=d" (reg2) + : "d" (reg0) + : "cc"); + if (info) + *info = reg2; + return reg1; +} + +/** * ap_test_queue(): Test adjunct processor queue. * @qid: The AP queue number * @tbit: Test facilities bit @@ -54,10 +98,57 @@ struct ap_queue_status { * * Returns AP queue status structure. */ -struct ap_queue_status ap_test_queue(ap_qid_t qid, - int tbit, - unsigned long *info); +static inline struct ap_queue_status ap_test_queue(ap_qid_t qid, + int tbit, + unsigned long *info) +{ + if (tbit) + qid |= 1UL << 23; /* set T bit*/ + return ap_tapq(qid, info); +} +/** + * ap_pqap_rapq(): Reset adjunct processor queue. + * @qid: The AP queue number + * + * Returns AP queue status structure. + */ +static inline struct ap_queue_status ap_rapq(ap_qid_t qid) +{ + register unsigned long reg0 asm ("0") = qid | (1UL << 24); + register struct ap_queue_status reg1 asm ("1"); + + asm volatile( + ".long 0xb2af0000" /* PQAP(RAPQ) */ + : "=d" (reg1) + : "d" (reg0) + : "cc"); + return reg1; +} + +/** + * ap_pqap_zapq(): Reset and zeroize adjunct processor queue. + * @qid: The AP queue number + * + * Returns AP queue status structure. + */ +static inline struct ap_queue_status ap_zapq(ap_qid_t qid) +{ + register unsigned long reg0 asm ("0") = qid | (2UL << 24); + register struct ap_queue_status reg1 asm ("1"); + + asm volatile( + ".long 0xb2af0000" /* PQAP(ZAPQ) */ + : "=d" (reg1) + : "d" (reg0) + : "cc"); + return reg1; +} + +/** + * struct ap_config_info - convenience struct for AP crypto + * config info as returned by the ap_qci() function. + */ struct ap_config_info { unsigned int apsc : 1; /* S bit */ unsigned int apxa : 1; /* N bit */ @@ -74,50 +165,189 @@ struct ap_config_info { unsigned char _reserved4[16]; } __aligned(8); -/* - * ap_query_configuration(): Fetch cryptographic config info +/** + * ap_qci(): Get AP configuration data * - * Returns the ap configuration info fetched via PQAP(QCI). - * On success 0 is returned, on failure a negative errno - * is returned, e.g. if the PQAP(QCI) instruction is not - * available, the return value will be -EOPNOTSUPP. + * Returns 0 on success, or -EOPNOTSUPP. */ -int ap_query_configuration(struct ap_config_info *info); +static inline int ap_qci(struct ap_config_info *config) +{ + register unsigned long reg0 asm ("0") = 4UL << 24; + register unsigned long reg1 asm ("1") = -EOPNOTSUPP; + register struct ap_config_info *reg2 asm ("2") = config; + + asm volatile( + ".long 0xb2af0000\n" /* PQAP(QCI) */ + "0: la %0,0\n" + "1:\n" + EX_TABLE(0b, 1b) + : "+d" (reg1) + : "d" (reg0), "d" (reg2) + : "cc", "memory"); + + return reg1; +} /* * struct ap_qirq_ctrl - convenient struct for easy invocation - * of the ap_queue_irq_ctrl() function. This struct is passed - * as GR1 parameter to the PQAP(AQIC) instruction. For details - * please see the AR documentation. + * of the ap_aqic() function. This struct is passed as GR1 + * parameter to the PQAP(AQIC) instruction. For details please + * see the AR documentation. */ struct ap_qirq_ctrl { unsigned int _res1 : 8; - unsigned int zone : 8; /* zone info */ - unsigned int ir : 1; /* ir flag: enable (1) or disable (0) irq */ + unsigned int zone : 8; /* zone info */ + unsigned int ir : 1; /* ir flag: enable (1) or disable (0) irq */ unsigned int _res2 : 4; - unsigned int gisc : 3; /* guest isc field */ + unsigned int gisc : 3; /* guest isc field */ unsigned int _res3 : 6; - unsigned int gf : 2; /* gisa format */ + unsigned int gf : 2; /* gisa format */ unsigned int _res4 : 1; - unsigned int gisa : 27; /* gisa origin */ + unsigned int gisa : 27; /* gisa origin */ unsigned int _res5 : 1; - unsigned int isc : 3; /* irq sub class */ + unsigned int isc : 3; /* irq sub class */ }; /** - * ap_queue_irq_ctrl(): Control interruption on a AP queue. + * ap_aqic(): Control interruption for a specific AP. * @qid: The AP queue number - * @qirqctrl: struct ap_qirq_ctrl, see above + * @qirqctrl: struct ap_qirq_ctrl (64 bit value) * @ind: The notification indicator byte * * Returns AP queue status. + */ +static inline struct ap_queue_status ap_aqic(ap_qid_t qid, + struct ap_qirq_ctrl qirqctrl, + void *ind) +{ + register unsigned long reg0 asm ("0") = qid | (3UL << 24); + register struct ap_qirq_ctrl reg1_in asm ("1") = qirqctrl; + register struct ap_queue_status reg1_out asm ("1"); + register void *reg2 asm ("2") = ind; + + asm volatile( + ".long 0xb2af0000" /* PQAP(AQIC) */ + : "=d" (reg1_out) + : "d" (reg0), "d" (reg1_in), "d" (reg2) + : "cc"); + return reg1_out; +} + +/* + * union ap_qact_ap_info - used together with the + * ap_aqic() function to provide a convenient way + * to handle the ap info needed by the qact function. + */ +union ap_qact_ap_info { + unsigned long val; + struct { + unsigned int : 3; + unsigned int mode : 3; + unsigned int : 26; + unsigned int cat : 8; + unsigned int : 8; + unsigned char ver[2]; + }; +}; + +/** + * ap_qact(): Query AP combatibility type. + * @qid: The AP queue number + * @apinfo: On input the info about the AP queue. On output the + * alternate AP queue info provided by the qact function + * in GR2 is stored in. * - * Control interruption on the given AP queue. - * Just a simple wrapper function for the low level PQAP(AQIC) - * instruction available for other kernel modules. + * Returns AP queue status. Check response_code field for failures. */ -struct ap_queue_status ap_queue_irq_ctrl(ap_qid_t qid, - struct ap_qirq_ctrl qirqctrl, - void *ind); +static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit, + union ap_qact_ap_info *apinfo) +{ + register unsigned long reg0 asm ("0") = qid | (5UL << 24) + | ((ifbit & 0x01) << 22); + register unsigned long reg1_in asm ("1") = apinfo->val; + register struct ap_queue_status reg1_out asm ("1"); + register unsigned long reg2 asm ("2"); + + asm volatile( + ".long 0xb2af0000" /* PQAP(QACT) */ + : "+d" (reg1_in), "=d" (reg1_out), "=d" (reg2) + : "d" (reg0) + : "cc"); + apinfo->val = reg2; + return reg1_out; +} + +/** + * ap_nqap(): Send message to adjunct processor queue. + * @qid: The AP queue number + * @psmid: The program supplied message identifier + * @msg: The message text + * @length: The message length + * + * Returns AP queue status structure. + * Condition code 1 on NQAP can't happen because the L bit is 1. + * Condition code 2 on NQAP also means the send is incomplete, + * because a segment boundary was reached. The NQAP is repeated. + */ +static inline struct ap_queue_status ap_nqap(ap_qid_t qid, + unsigned long long psmid, + void *msg, size_t length) +{ + register unsigned long reg0 asm ("0") = qid | 0x40000000UL; + register struct ap_queue_status reg1 asm ("1"); + register unsigned long reg2 asm ("2") = (unsigned long) msg; + register unsigned long reg3 asm ("3") = (unsigned long) length; + register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32); + register unsigned long reg5 asm ("5") = psmid & 0xffffffff; + + asm volatile ( + "0: .long 0xb2ad0042\n" /* NQAP */ + " brc 2,0b" + : "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3) + : "d" (reg4), "d" (reg5) + : "cc", "memory"); + return reg1; +} + +/** + * ap_dqap(): Receive message from adjunct processor queue. + * @qid: The AP queue number + * @psmid: Pointer to program supplied message identifier + * @msg: The message text + * @length: The message length + * + * Returns AP queue status structure. + * Condition code 1 on DQAP means the receive has taken place + * but only partially. The response is incomplete, hence the + * DQAP is repeated. + * Condition code 2 on DQAP also means the receive is incomplete, + * this time because a segment boundary was reached. Again, the + * DQAP is repeated. + * Note that gpr2 is used by the DQAP instruction to keep track of + * any 'residual' length, in case the instruction gets interrupted. + * Hence it gets zeroed before the instruction. + */ +static inline struct ap_queue_status ap_dqap(ap_qid_t qid, + unsigned long long *psmid, + void *msg, size_t length) +{ + register unsigned long reg0 asm("0") = qid | 0x80000000UL; + register struct ap_queue_status reg1 asm ("1"); + register unsigned long reg2 asm("2") = 0UL; + register unsigned long reg4 asm("4") = (unsigned long) msg; + register unsigned long reg5 asm("5") = (unsigned long) length; + register unsigned long reg6 asm("6") = 0UL; + register unsigned long reg7 asm("7") = 0UL; + + + asm volatile( + "0: .long 0xb2ae0064\n" /* DQAP */ + " brc 6,0b\n" + : "+d" (reg0), "=d" (reg1), "+d" (reg2), + "+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7) + : : "cc", "memory"); + *psmid = (((unsigned long long) reg6) << 32) + reg7; + return reg1; +} #endif /* _ASM_S390_AP_H_ */ diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 4b55532f15c4..fd20ab5d4cf7 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -55,17 +55,9 @@ static inline void atomic_add(int i, atomic_t *v) __atomic_add(i, &v->counter); } -#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) -#define atomic_inc(_v) atomic_add(1, _v) -#define atomic_inc_return(_v) atomic_add_return(1, _v) -#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) #define atomic_sub(_i, _v) atomic_add(-(int)(_i), _v) #define atomic_sub_return(_i, _v) atomic_add_return(-(int)(_i), _v) #define atomic_fetch_sub(_i, _v) atomic_fetch_add(-(int)(_i), _v) -#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) -#define atomic_dec(_v) atomic_sub(1, _v) -#define atomic_dec_return(_v) atomic_sub_return(1, _v) -#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) #define ATOMIC_OPS(op) \ static inline void atomic_##op(int i, atomic_t *v) \ @@ -90,21 +82,6 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) return __atomic_cmpxchg(&v->counter, old, new); } -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == u)) - break; - old = atomic_cmpxchg(v, c, c + a); - if (likely(old == c)) - break; - c = old; - } - return c; -} - #define ATOMIC64_INIT(i) { (i) } static inline long atomic64_read(const atomic64_t *v) @@ -168,50 +145,8 @@ ATOMIC64_OPS(xor) #undef ATOMIC64_OPS -static inline int atomic64_add_unless(atomic64_t *v, long i, long u) -{ - long c, old; - - c = atomic64_read(v); - for (;;) { - if (unlikely(c == u)) - break; - old = atomic64_cmpxchg(v, c, c + i); - if (likely(old == c)) - break; - c = old; - } - return c != u; -} - -static inline long atomic64_dec_if_positive(atomic64_t *v) -{ - long c, old, dec; - - c = atomic64_read(v); - for (;;) { - dec = c - 1; - if (unlikely(dec < 0)) - break; - old = atomic64_cmpxchg((v), c, dec); - if (likely(old == c)) - break; - c = old; - } - return dec; -} - -#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) -#define atomic64_inc(_v) atomic64_add(1, _v) -#define atomic64_inc_return(_v) atomic64_add_return(1, _v) -#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) #define atomic64_sub_return(_i, _v) atomic64_add_return(-(long)(_i), _v) #define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long)(_i), _v) #define atomic64_sub(_i, _v) atomic64_add(-(long)(_i), _v) -#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) -#define atomic64_dec(_v) atomic64_sub(1, _v) -#define atomic64_dec_return(_v) atomic64_sub_return(1, _v) -#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #endif /* __ARCH_S390_ATOMIC__ */ diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index de023a9a88ca..bf2cbff926ef 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -2,7 +2,7 @@ /* * CPU-measurement facilities * - * Copyright IBM Corp. 2012 + * Copyright IBM Corp. 2012, 2018 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> * Jan Glauber <jang@linux.vnet.ibm.com> */ @@ -139,8 +139,14 @@ struct hws_trailer_entry { unsigned char timestamp[16]; /* 16 - 31 timestamp */ unsigned long long reserved1; /* 32 -Reserved */ unsigned long long reserved2; /* */ - unsigned long long progusage1; /* 48 - reserved for programming use */ - unsigned long long progusage2; /* */ + union { /* 48 - reserved for programming use */ + struct { + unsigned int clock_base:1; /* in progusage2 */ + unsigned long long progusage1:63; + unsigned long long progusage2; + }; + unsigned long long progusage[2]; + }; } __packed; /* Load program parameter */ diff --git a/arch/s390/include/asm/gmap.h b/arch/s390/include/asm/gmap.h index e07cce88dfb0..fcbd638fb9f4 100644 --- a/arch/s390/include/asm/gmap.h +++ b/arch/s390/include/asm/gmap.h @@ -9,6 +9,14 @@ #ifndef _ASM_S390_GMAP_H #define _ASM_S390_GMAP_H +/* Generic bits for GMAP notification on DAT table entry changes. */ +#define GMAP_NOTIFY_SHADOW 0x2 +#define GMAP_NOTIFY_MPROT 0x1 + +/* Status bits only for huge segment entries */ +#define _SEGMENT_ENTRY_GMAP_IN 0x8000 /* invalidation notify bit */ +#define _SEGMENT_ENTRY_GMAP_UC 0x4000 /* dirty (migration) */ + /** * struct gmap_struct - guest address space * @list: list head for the mm->context gmap list @@ -132,4 +140,6 @@ void gmap_pte_notify(struct mm_struct *, unsigned long addr, pte_t *, int gmap_mprotect_notify(struct gmap *, unsigned long start, unsigned long len, int prot); +void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long dirty_bitmap[4], + unsigned long gaddr, unsigned long vmaddr); #endif /* _ASM_S390_GMAP_H */ diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index 9c5fc50204dd..2d1afa58a4b6 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -37,7 +37,10 @@ static inline int prepare_hugepage_range(struct file *file, return 0; } -#define arch_clear_hugepage_flags(page) do { } while (0) +static inline void arch_clear_hugepage_flags(struct page *page) +{ + clear_bit(PG_arch_1, &page->flags); +} static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long sz) diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index 13de80cf741c..b106aa29bf55 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -68,8 +68,6 @@ struct kprobe_ctlblk { unsigned long kprobe_saved_imask; unsigned long kprobe_saved_ctl[3]; struct prev_kprobe prev_kprobe; - struct pt_regs jprobe_saved_regs; - kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE]; }; void arch_remove_kprobe(struct kprobe *p); diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5bc888841eaf..406d940173ab 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -185,7 +185,7 @@ struct lowcore { /* Transaction abort diagnostic block */ __u8 pgm_tdb[256]; /* 0x1800 */ __u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */ -} __packed; +} __packed __aligned(8192); #define S390_lowcore (*((struct lowcore *) 0)) diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index f5ff9dbad8ac..f31a15044c24 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -24,6 +24,8 @@ typedef struct { unsigned int uses_skeys:1; /* The mmu context uses CMM. */ unsigned int uses_cmm:1; + /* The gmaps associated with this context are allowed to use huge pages. */ + unsigned int allow_gmap_hpage_1m:1; } mm_context_t; #define INIT_MM_CONTEXT(name) \ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index d16bc79c30bb..0717ee76885d 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -32,6 +32,7 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.has_pgste = 0; mm->context.uses_skeys = 0; mm->context.uses_cmm = 0; + mm->context.allow_gmap_hpage_1m = 0; #endif switch (mm->context.asce_limit) { case _REGION2_SIZE: diff --git a/arch/s390/include/asm/nospec-insn.h b/arch/s390/include/asm/nospec-insn.h index a01f81186e86..123dac3717b3 100644 --- a/arch/s390/include/asm/nospec-insn.h +++ b/arch/s390/include/asm/nospec-insn.h @@ -8,7 +8,7 @@ #ifdef __ASSEMBLY__ -#ifdef CONFIG_EXPOLINE +#ifdef CC_USING_EXPOLINE _LC_BR_R1 = __LC_BR_R1 @@ -189,7 +189,7 @@ _LC_BR_R1 = __LC_BR_R1 .macro BASR_EX rsave,rtarget,ruse=%r1 basr \rsave,\rtarget .endm -#endif +#endif /* CC_USING_EXPOLINE */ #endif /* __ASSEMBLY__ */ diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 94f8db468c9b..10fe982f2b4b 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -51,6 +51,10 @@ struct zpci_fmb_fmt2 { u64 max_work_units; }; +struct zpci_fmb_fmt3 { + u64 tx_bytes; +}; + struct zpci_fmb { u32 format : 8; u32 fmt_ind : 24; @@ -66,6 +70,7 @@ struct zpci_fmb { struct zpci_fmb_fmt0 fmt0; struct zpci_fmb_fmt1 fmt1; struct zpci_fmb_fmt2 fmt2; + struct zpci_fmb_fmt3 fmt3; }; } __packed __aligned(128); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 5ab636089c60..0e7cb0dc9c33 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -268,8 +268,10 @@ static inline int is_module_addr(void *addr) #define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe2fUL /* Bits in the segment table entry */ -#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL -#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL +#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL +#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL +#define _SEGMENT_ENTRY_HARDWARE_BITS 0xfffffffffffffe30UL +#define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE 0xfffffffffff00730UL #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */ #define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* page table origin */ #define _SEGMENT_ENTRY_PROTECT 0x200 /* segment protection bit */ @@ -1101,7 +1103,8 @@ int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *sptep, pte_t *tptep, pte_t pte); void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep); -bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address); +bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address, + pte_t *ptep); int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, unsigned char key, bool nq); int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, @@ -1116,6 +1119,10 @@ int set_pgste_bits(struct mm_struct *mm, unsigned long addr, int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep); int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, unsigned long *oldpte, unsigned long *oldpgste); +void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr); +void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr); +void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr); +void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr); /* * Certain architectures need to do special things when PTEs diff --git a/arch/s390/include/asm/purgatory.h b/arch/s390/include/asm/purgatory.h index 6090670df51f..e297bcfc476f 100644 --- a/arch/s390/include/asm/purgatory.h +++ b/arch/s390/include/asm/purgatory.h @@ -13,11 +13,5 @@ int verify_sha256_digest(void); -extern u64 kernel_entry; -extern u64 kernel_type; - -extern u64 crash_start; -extern u64 crash_size; - #endif /* __ASSEMBLY__ */ #endif /* _S390_PURGATORY_H_ */ diff --git a/arch/s390/include/asm/qdio.h b/arch/s390/include/asm/qdio.h index de11ecc99c7c..9c9970a5dfb1 100644 --- a/arch/s390/include/asm/qdio.h +++ b/arch/s390/include/asm/qdio.h @@ -262,7 +262,6 @@ struct qdio_outbuf_state { void *user; }; -#define QDIO_OUTBUF_STATE_FLAG_NONE 0x00 #define QDIO_OUTBUF_STATE_FLAG_PENDING 0x01 #define CHSC_AC1_INITIATE_INPUTQ 0x80 diff --git a/arch/s390/include/asm/sections.h b/arch/s390/include/asm/sections.h index 54f81f8ed662..724faede8ac5 100644 --- a/arch/s390/include/asm/sections.h +++ b/arch/s390/include/asm/sections.h @@ -4,6 +4,4 @@ #include <asm-generic/sections.h> -extern char _ehead[]; - #endif diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 9c30ebe046f3..1d66016f4170 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -9,8 +9,10 @@ #include <linux/const.h> #include <uapi/asm/setup.h> - +#define EP_OFFSET 0x10008 +#define EP_STRING "S390EP" #define PARMAREA 0x10400 +#define PARMAREA_END 0x11000 /* * Machine features detected in early.c diff --git a/arch/s390/include/uapi/asm/chsc.h b/arch/s390/include/uapi/asm/chsc.h index dc329aa03f76..83a574e95b3a 100644 --- a/arch/s390/include/uapi/asm/chsc.h +++ b/arch/s390/include/uapi/asm/chsc.h @@ -23,29 +23,29 @@ struct chsc_async_header { __u32 key : 4; __u32 : 28; struct subchannel_id sid; -} __attribute__ ((packed)); +}; struct chsc_async_area { struct chsc_async_header header; __u8 data[CHSC_SIZE - sizeof(struct chsc_async_header)]; -} __attribute__ ((packed)); +}; struct chsc_header { __u16 length; __u16 code; -} __attribute__ ((packed)); +}; struct chsc_sync_area { struct chsc_header header; __u8 data[CHSC_SIZE - sizeof(struct chsc_header)]; -} __attribute__ ((packed)); +}; struct chsc_response_struct { __u16 length; __u16 code; __u32 parms; __u8 data[CHSC_SIZE - 2 * sizeof(__u16) - sizeof(__u32)]; -} __attribute__ ((packed)); +}; struct chsc_chp_cd { struct chp_id chpid; diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 2fed39b26b42..dbfd1730e631 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -9,39 +9,21 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) # Do not trace early setup code -CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_early_nobss.o = $(CC_FLAGS_FTRACE) endif -GCOV_PROFILE_als.o := n GCOV_PROFILE_early.o := n GCOV_PROFILE_early_nobss.o := n -KCOV_INSTRUMENT_als.o := n KCOV_INSTRUMENT_early.o := n KCOV_INSTRUMENT_early_nobss.o := n -UBSAN_SANITIZE_als.o := n UBSAN_SANITIZE_early.o := n UBSAN_SANITIZE_early_nobss.o := n # -# Use -march=z900 for als.c to be able to print an error -# message if the kernel is started on a machine which is too old -# -ifneq ($(CC_FLAGS_MARCH),-march=z900) -CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) -CFLAGS_REMOVE_als.o += $(CC_FLAGS_EXPOLINE) -CFLAGS_als.o += -march=z900 -AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) -AFLAGS_head.o += -march=z900 -endif - -CFLAGS_als.o += -D__NO_FORTIFY - -# # Passing null pointers is ok for smp code, since we access the lowcore here. # CFLAGS_smp.o := -Wno-nonnull @@ -61,13 +43,13 @@ CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o -obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o als.o early_nobss.o +obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o early_nobss.o obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o -extra-y += head.o head64.o vmlinux.lds +extra-y += head64.o vmlinux.lds obj-$(CONFIG_SYSFS) += nospec-sysfs.o CFLAGS_REMOVE_nospec-branch.o += $(CC_FLAGS_EXPOLINE) @@ -99,5 +81,5 @@ obj-$(CONFIG_TRACEPOINTS) += trace.o obj-y += vdso64/ obj-$(CONFIG_COMPAT) += vdso32/ -chkbss := head.o head64.o als.o early_nobss.o +chkbss := head64.o early_nobss.o include $(srctree)/arch/s390/scripts/Makefile.chkbss diff --git a/arch/s390/kernel/compat_wrapper.c b/arch/s390/kernel/compat_wrapper.c index 607c5e9fba3d..2ce28bf0c5ec 100644 --- a/arch/s390/kernel/compat_wrapper.c +++ b/arch/s390/kernel/compat_wrapper.c @@ -183,3 +183,4 @@ COMPAT_SYSCALL_WRAP2(s390_guarded_storage, int, command, struct gs_cb *, gs_cb); COMPAT_SYSCALL_WRAP5(statx, int, dfd, const char __user *, path, unsigned, flags, unsigned, mask, struct statx __user *, buffer); COMPAT_SYSCALL_WRAP4(s390_sthyi, unsigned long, code, void __user *, info, u64 __user *, rc, unsigned long, flags); COMPAT_SYSCALL_WRAP5(kexec_file_load, int, kernel_fd, int, initrd_fd, unsigned long, cmdline_len, const char __user *, cmdline_ptr, unsigned long, flags) +COMPAT_SYSCALL_WRAP4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 9f5ea9d87069..c3620bafc374 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -306,6 +306,15 @@ static void *kzalloc_panic(int len) return rc; } +static const char *nt_name(Elf64_Word type) +{ + const char *name = "LINUX"; + + if (type == NT_PRPSINFO || type == NT_PRSTATUS || type == NT_PRFPREG) + name = KEXEC_CORE_NOTE_NAME; + return name; +} + /* * Initialize ELF note */ @@ -332,11 +341,26 @@ static void *nt_init_name(void *buf, Elf64_Word type, void *desc, int d_len, static inline void *nt_init(void *buf, Elf64_Word type, void *desc, int d_len) { - const char *note_name = "LINUX"; + return nt_init_name(buf, type, desc, d_len, nt_name(type)); +} + +/* + * Calculate the size of ELF note + */ +static size_t nt_size_name(int d_len, const char *name) +{ + size_t size; - if (type == NT_PRPSINFO || type == NT_PRSTATUS || type == NT_PRFPREG) - note_name = KEXEC_CORE_NOTE_NAME; - return nt_init_name(buf, type, desc, d_len, note_name); + size = sizeof(Elf64_Nhdr); + size += roundup(strlen(name) + 1, 4); + size += roundup(d_len, 4); + + return size; +} + +static inline size_t nt_size(Elf64_Word type, int d_len) +{ + return nt_size_name(d_len, nt_name(type)); } /* @@ -375,6 +399,29 @@ static void *fill_cpu_elf_notes(void *ptr, int cpu, struct save_area *sa) } /* + * Calculate size of ELF notes per cpu + */ +static size_t get_cpu_elf_notes_size(void) +{ + struct save_area *sa = NULL; + size_t size; + + size = nt_size(NT_PRSTATUS, sizeof(struct elf_prstatus)); + size += nt_size(NT_PRFPREG, sizeof(elf_fpregset_t)); + size += nt_size(NT_S390_TIMER, sizeof(sa->timer)); + size += nt_size(NT_S390_TODCMP, sizeof(sa->todcmp)); + size += nt_size(NT_S390_TODPREG, sizeof(sa->todpreg)); + size += nt_size(NT_S390_CTRS, sizeof(sa->ctrs)); + size += nt_size(NT_S390_PREFIX, sizeof(sa->prefix)); + if (MACHINE_HAS_VX) { + size += nt_size(NT_S390_VXRS_HIGH, sizeof(sa->vxrs_high)); + size += nt_size(NT_S390_VXRS_LOW, sizeof(sa->vxrs_low)); + } + + return size; +} + +/* * Initialize prpsinfo note (new kernel) */ static void *nt_prpsinfo(void *ptr) @@ -429,6 +476,30 @@ static void *nt_vmcoreinfo(void *ptr) return nt_init_name(ptr, 0, vmcoreinfo, size, "VMCOREINFO"); } +static size_t nt_vmcoreinfo_size(void) +{ + const char *name = "VMCOREINFO"; + char nt_name[11]; + Elf64_Nhdr note; + void *addr; + + if (copy_oldmem_kernel(&addr, &S390_lowcore.vmcore_info, sizeof(addr))) + return 0; + + if (copy_oldmem_kernel(¬e, addr, sizeof(note))) + return 0; + + memset(nt_name, 0, sizeof(nt_name)); + if (copy_oldmem_kernel(nt_name, addr + sizeof(note), + sizeof(nt_name) - 1)) + return 0; + + if (strcmp(nt_name, name) != 0) + return 0; + + return nt_size_name(note.n_descsz, name); +} + /* * Initialize final note (needed for /proc/vmcore code) */ @@ -539,6 +610,27 @@ static void *notes_init(Elf64_Phdr *phdr, void *ptr, u64 notes_offset) return ptr; } +static size_t get_elfcorehdr_size(int mem_chunk_cnt) +{ + size_t size; + + size = sizeof(Elf64_Ehdr); + /* PT_NOTES */ + size += sizeof(Elf64_Phdr); + /* nt_prpsinfo */ + size += nt_size(NT_PRPSINFO, sizeof(struct elf_prpsinfo)); + /* regsets */ + size += get_cpu_cnt() * get_cpu_elf_notes_size(); + /* nt_vmcoreinfo */ + size += nt_vmcoreinfo_size(); + /* nt_final */ + size += sizeof(Elf64_Nhdr); + /* PT_LOADS */ + size += mem_chunk_cnt * sizeof(Elf64_Phdr); + + return size; +} + /* * Create ELF core header (new kernel) */ @@ -566,8 +658,8 @@ int elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size) mem_chunk_cnt = get_mem_chunk_cnt(); - alloc_size = 0x1000 + get_cpu_cnt() * 0x4a0 + - mem_chunk_cnt * sizeof(Elf64_Phdr); + alloc_size = get_elfcorehdr_size(mem_chunk_cnt); + hdr = kzalloc_panic(alloc_size); /* Init elf header */ ptr = ehdr_init(hdr, mem_chunk_cnt); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 827699eb48fa..5b28b434f8a1 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -331,8 +331,20 @@ static void __init setup_boot_command_line(void) append_to_cmdline(append_ipl_scpdata); } +static void __init check_image_bootable(void) +{ + if (!memcmp(EP_STRING, (void *)EP_OFFSET, strlen(EP_STRING))) + return; + + sclp_early_printk("Linux kernel boot failure: An attempt to boot a vmlinux ELF image failed.\n"); + sclp_early_printk("This image does not contain all parts necessary for starting up. Use\n"); + sclp_early_printk("bzImage or arch/s390/boot/compressed/vmlinux instead.\n"); + disabled_wait(0xbadb007); +} + void __init startup_init(void) { + check_image_bootable(); time_early_init(); init_kernel_storage_key(); lockdep_off(); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index f03402efab4b..150130c897c3 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -357,6 +357,10 @@ ENTRY(system_call) stg %r2,__PT_R2(%r11) # store return value .Lsysc_return: +#ifdef CONFIG_DEBUG_RSEQ + lgr %r2,%r11 + brasl %r14,rseq_syscall +#endif LOCKDEP_SYS_EXIT .Lsysc_tif: TSTMSK __PT_FLAGS(%r11),_PIF_WORK @@ -1265,7 +1269,7 @@ cleanup_critical: jl 0f clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end jl .Lcleanup_load_fpu_regs -0: BR_EX %r14 +0: BR_EX %r14,%r11 .align 8 .Lcleanup_table: @@ -1301,7 +1305,7 @@ cleanup_critical: ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE lctlg %c1,%c1,__LC_USER_ASCE # load primary asce larl %r9,sie_exit # skip forward to sie_exit - BR_EX %r14 + BR_EX %r14,%r11 #endif .Lcleanup_system_call: diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 961abfac2c5f..472fa2f1a4a5 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -83,7 +83,6 @@ long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user DECLARE_PER_CPU(u64, mt_cycles[8]); -void verify_facilities(void); void gs_load_bc_cb(struct pt_regs *regs); void set_fs_fixup(void); diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 791cb9000e86..6d14ad42ba88 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -48,11 +48,23 @@ ENTRY(startup_continue) # Early machine initialization and detection functions. # brasl %r14,startup_init - lpswe .Lentry-.LPG1(13) # jump to _stext in primary-space, - # virtual and never return ... + +# check control registers + stctg %c0,%c15,0(%r15) + oi 6(%r15),0x60 # enable sigp emergency & external call + oi 4(%r15),0x10 # switch on low address proctection + lctlg %c0,%c15,0(%r15) + + lam 0,15,.Laregs-.LPG1(%r13) # load acrs needed by uaccess + brasl %r14,start_kernel # go to C code +# +# We returned from start_kernel ?!? PANIK +# + basr %r13,0 + lpswe .Ldw-.(%r13) # load disabled wait psw + .align 16 .LPG1: -.Lentry:.quad 0x0000000180000000,_stext .Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space .quad 0 # cr1: primary space segment table .quad .Lduct # cr2: dispatchable unit control table @@ -85,30 +97,5 @@ ENTRY(startup_continue) .endr .Llinkage_stack: .long 0,0,0x89000000,0,0,0,0x8a000000,0 - -ENTRY(_ehead) - - .org 0x100000 - 0x11000 # head.o ends at 0x11000 -# -# startup-code, running in absolute addressing mode -# -ENTRY(_stext) - basr %r13,0 # get base -.LPG3: -# check control registers - stctg %c0,%c15,0(%r15) - oi 6(%r15),0x60 # enable sigp emergency & external call - oi 4(%r15),0x10 # switch on low address proctection - lctlg %c0,%c15,0(%r15) - - lam 0,15,.Laregs-.LPG3(%r13) # load acrs needed by uaccess - brasl %r14,start_kernel # go to C code -# -# We returned from start_kernel ?!? PANIK -# - basr %r13,0 - lpswe .Ldw-.(%r13) # load disabled wait psw - - .align 8 .Ldw: .quad 0x0002000180000000,0x0000000000000000 .Laregs:.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 60f60afa645c..7c0a095e9c5f 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -321,38 +321,20 @@ static int kprobe_handler(struct pt_regs *regs) * If we have no pre-handler or it returned 0, we * continue with single stepping. If we have a * pre-handler and it returned non-zero, it prepped - * for calling the break_handler below on re-entry - * for jprobe processing, so get out doing nothing - * more here. + * for changing execution path, so get out doing + * nothing more here. */ push_kprobe(kcb, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (p->pre_handler && p->pre_handler(p, regs)) + if (p->pre_handler && p->pre_handler(p, regs)) { + pop_kprobe(kcb); + preempt_enable_no_resched(); return 1; + } kcb->kprobe_status = KPROBE_HIT_SS; } enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn); return 1; - } else if (kprobe_running()) { - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - /* - * Continuation after the jprobe completed and - * caused the jprobe_return trap. The jprobe - * break_handler "returns" to the original - * function that still has the kprobe breakpoint - * installed. We continue with single stepping. - */ - kcb->kprobe_status = KPROBE_HIT_SS; - enable_singlestep(kcb, regs, - (unsigned long) p->ainsn.insn); - return 1; - } /* else: - * No kprobe at this address and the current kprobe - * has no break handler (no jprobe!). The kernel just - * exploded, let the standard trap handler pick up the - * pieces. - */ } /* else: * No kprobe at this address and no active kprobe. The trap has * not been caused by a kprobe breakpoint. The race of breakpoint @@ -452,9 +434,7 @@ static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) regs->psw.addr = orig_ret_address; - pop_kprobe(get_kprobe_ctlblk()); kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); @@ -661,60 +641,6 @@ int kprobe_exceptions_notify(struct notifier_block *self, } NOKPROBE_SYMBOL(kprobe_exceptions_notify); -int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long stack; - - memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); - - /* setup return addr to the jprobe handler routine */ - regs->psw.addr = (unsigned long) jp->entry; - regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); - - /* r15 is the stack pointer */ - stack = (unsigned long) regs->gprs[15]; - - memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack)); - - /* - * jprobes use jprobe_return() which skips the normal return - * path of the function, and this messes up the accounting of the - * function graph tracer to get messed up. - * - * Pause function graph tracing while performing the jprobe function. - */ - pause_graph_tracing(); - return 1; -} -NOKPROBE_SYMBOL(setjmp_pre_handler); - -void jprobe_return(void) -{ - asm volatile(".word 0x0002"); -} -NOKPROBE_SYMBOL(jprobe_return); - -int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long stack; - - /* It's OK to start function graph tracing again */ - unpause_graph_tracing(); - - stack = (unsigned long) kcb->jprobe_saved_regs.gprs[15]; - - /* Put the regs back */ - memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); - /* put the stack back */ - memcpy((void *) stack, kcb->jprobes_stack, MIN_STACK_SIZE(stack)); - preempt_enable_no_resched(); - return 1; -} -NOKPROBE_SYMBOL(longjmp_break_handler); - static struct kprobe trampoline = { .addr = (kprobe_opcode_t *) &kretprobe_trampoline, .pre_handler = trampoline_probe_handler diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c index 18ae7b9c71d6..bdddaae96559 100644 --- a/arch/s390/kernel/nospec-branch.c +++ b/arch/s390/kernel/nospec-branch.c @@ -35,6 +35,8 @@ early_param("nospec", nospec_setup_early); static int __init nospec_report(void) { + if (test_facility(156)) + pr_info("Spectre V2 mitigation: etokens\n"); if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) pr_info("Spectre V2 mitigation: execute trampolines\n"); if (__test_facility(82, S390_lowcore.alt_stfle_fac_list)) @@ -56,7 +58,15 @@ early_param("nospectre_v2", nospectre_v2_setup_early); void __init nospec_auto_detect(void) { - if (IS_ENABLED(CC_USING_EXPOLINE)) { + if (test_facility(156)) { + /* + * The machine supports etokens. + * Disable expolines and disable nobp. + */ + if (IS_ENABLED(CC_USING_EXPOLINE)) + nospec_disable = 1; + __clear_facility(82, S390_lowcore.alt_stfle_fac_list); + } else if (IS_ENABLED(CC_USING_EXPOLINE)) { /* * The kernel has been compiled with expolines. * Keep expolines enabled and disable nobp. diff --git a/arch/s390/kernel/nospec-sysfs.c b/arch/s390/kernel/nospec-sysfs.c index 8affad5f18cb..e30e580ae362 100644 --- a/arch/s390/kernel/nospec-sysfs.c +++ b/arch/s390/kernel/nospec-sysfs.c @@ -13,6 +13,8 @@ ssize_t cpu_show_spectre_v1(struct device *dev, ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) { + if (test_facility(156)) + return sprintf(buf, "Mitigation: etokens\n"); if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) return sprintf(buf, "Mitigation: execute trampolines\n"); if (__test_facility(82, S390_lowcore.alt_stfle_fac_list)) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 0292d68e7dde..cb198d4a6dca 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -2,7 +2,7 @@ /* * Performance event support for the System z CPU-measurement Sampling Facility * - * Copyright IBM Corp. 2013 + * Copyright IBM Corp. 2013, 2018 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> */ #define KMSG_COMPONENT "cpum_sf" @@ -1587,6 +1587,17 @@ static void aux_buffer_free(void *data) "%lu SDBTs\n", num_sdbt); } +static void aux_sdb_init(unsigned long sdb) +{ + struct hws_trailer_entry *te; + + te = (struct hws_trailer_entry *)trailer_entry_ptr(sdb); + + /* Save clock base */ + te->clock_base = 1; + memcpy(&te->progusage2, &tod_clock_base[1], 8); +} + /* * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling * @cpu: On which to allocate, -1 means current @@ -1666,6 +1677,7 @@ static void *aux_buffer_setup(int cpu, void **pages, int nr_pages, /* Tail is the entry in a SDBT */ *tail = (unsigned long)pages[i]; aux->sdb_index[i] = (unsigned long)pages[i]; + aux_sdb_init((unsigned long)pages[i]); } sfb->num_sdb = nr_pages; diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 54e2d634b849..4352a504f235 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -12,9 +12,6 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) { freg_t fp; - if (WARN_ON_ONCE((u32)idx >= PERF_REG_S390_MAX)) - return 0; - if (idx >= PERF_REG_S390_R0 && idx <= PERF_REG_S390_R15) return regs->gprs[idx]; @@ -33,7 +30,8 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) if (idx == PERF_REG_S390_PC) return regs->psw.addr; - return regs->gprs[idx]; + WARN_ON_ONCE((u32)idx >= PERF_REG_S390_MAX); + return 0; } #define REG_RESERVED (~((1UL << PERF_REG_S390_MAX) - 1)) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d82a9ec64ea9..c637c12f9e37 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -674,12 +674,12 @@ static void __init reserve_kernel(void) #ifdef CONFIG_DMA_API_DEBUG /* * DMA_API_DEBUG code stumbles over addresses from the - * range [_ehead, _stext]. Mark the memory as reserved + * range [PARMAREA_END, _stext]. Mark the memory as reserved * so it is not used for CONFIG_DMA_API_DEBUG=y. */ memblock_reserve(0, PFN_PHYS(start_pfn)); #else - memblock_reserve(0, (unsigned long)_ehead); + memblock_reserve(0, PARMAREA_END); memblock_reserve((unsigned long)_stext, PFN_PHYS(start_pfn) - (unsigned long)_stext); #endif diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 2d2960ab3e10..22f08245aa5d 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -498,7 +498,7 @@ void do_signal(struct pt_regs *regs) } /* No longer in a system call */ clear_pt_regs_flag(regs, PIF_SYSCALL); - + rseq_signal_deliver(&ksig, regs); if (is_compat_task()) handle_signal32(&ksig, oldset, regs); else @@ -537,4 +537,5 @@ void do_notify_resume(struct pt_regs *regs) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); + rseq_handle_notify_resume(NULL, regs); } diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index 8b210ead7956..022fc099b628 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -389,3 +389,5 @@ 379 common statx sys_statx compat_sys_statx 380 common s390_sthyi sys_s390_sthyi compat_sys_s390_sthyi 381 common kexec_file_load sys_kexec_file_load compat_sys_kexec_file_load +382 common io_pgetevents sys_io_pgetevents compat_sys_io_pgetevents +383 common rseq sys_rseq compat_sys_rseq diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c index 54f5496913fa..12f80d1f0415 100644 --- a/arch/s390/kernel/sysinfo.c +++ b/arch/s390/kernel/sysinfo.c @@ -59,6 +59,8 @@ int stsi(void *sysinfo, int fc, int sel1, int sel2) } EXPORT_SYMBOL(stsi); +#ifdef CONFIG_PROC_FS + static bool convert_ext_name(unsigned char encoding, char *name, size_t len) { switch (encoding) { @@ -301,6 +303,8 @@ static int __init sysinfo_create_proc(void) } device_initcall(sysinfo_create_proc); +#endif /* CONFIG_PROC_FS */ + /* * Service levels interface. */ diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index cf561160ea88..e8766beee5ad 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -221,17 +221,22 @@ void read_persistent_clock64(struct timespec64 *ts) ext_to_timespec64(clk, ts); } -void read_boot_clock64(struct timespec64 *ts) +void __init read_persistent_wall_and_boot_offset(struct timespec64 *wall_time, + struct timespec64 *boot_offset) { unsigned char clk[STORE_CLOCK_EXT_SIZE]; + struct timespec64 boot_time; __u64 delta; delta = initial_leap_seconds + TOD_UNIX_EPOCH; - memcpy(clk, tod_clock_base, 16); - *(__u64 *) &clk[1] -= delta; - if (*(__u64 *) &clk[1] > delta) + memcpy(clk, tod_clock_base, STORE_CLOCK_EXT_SIZE); + *(__u64 *)&clk[1] -= delta; + if (*(__u64 *)&clk[1] > delta) clk[0]--; - ext_to_timespec64(clk, ts); + ext_to_timespec64(clk, &boot_time); + + read_persistent_clock64(wall_time); + *boot_offset = timespec64_sub(*wall_time, boot_time); } static u64 read_tod_clock(struct clocksource *cs) diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 4b6e0397f66d..e8184a15578a 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -579,41 +579,33 @@ early_param("topology", topology_setup); static int topology_ctl_handler(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - unsigned int len; + int enabled = topology_is_enabled(); int new_mode; - char buf[2]; + int zero = 0; + int one = 1; + int rc; + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &enabled, + .maxlen = sizeof(int), + .extra1 = &zero, + .extra2 = &one, + }; + + rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; - if (!*lenp || *ppos) { - *lenp = 0; - return 0; - } - if (!write) { - strncpy(buf, topology_is_enabled() ? "1\n" : "0\n", - ARRAY_SIZE(buf)); - len = strnlen(buf, ARRAY_SIZE(buf)); - if (len > *lenp) - len = *lenp; - if (copy_to_user(buffer, buf, len)) - return -EFAULT; - goto out; - } - len = *lenp; - if (copy_from_user(buf, buffer, len > sizeof(buf) ? sizeof(buf) : len)) - return -EFAULT; - if (buf[0] != '0' && buf[0] != '1') - return -EINVAL; mutex_lock(&smp_cpu_state_mutex); - new_mode = topology_get_mode(buf[0] == '1'); + new_mode = topology_get_mode(enabled); if (topology_mode != new_mode) { topology_mode = new_mode; topology_schedule_update(); } mutex_unlock(&smp_cpu_state_mutex); topology_flush_work(); -out: - *lenp = len; - *ppos += len; - return 0; + + return rc; } static struct ctl_table topology_ctl_table[] = { diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 09abae40f917..3031cc6dd0ab 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -47,7 +47,7 @@ static struct page **vdso64_pagelist; */ unsigned int __read_mostly vdso_enabled = 1; -static int vdso_fault(const struct vm_special_mapping *sm, +static vm_fault_t vdso_fault(const struct vm_special_mapping *sm, struct vm_area_struct *vma, struct vm_fault *vmf) { struct page **vdso_pagelist; diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index f0414f52817b..b43f8d33a369 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -19,7 +19,7 @@ OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_ARCH(s390:64-bit) -ENTRY(startup) +ENTRY(startup_continue) jiffies = jiffies_64; PHDRS { @@ -30,16 +30,12 @@ PHDRS { SECTIONS { - . = 0x00000000; + . = 0x100000; + _stext = .; /* Start of text section */ .text : { /* Text and read-only data */ + _text = .; HEAD_TEXT - /* - * E.g. perf doesn't like symbols starting at address zero, - * therefore skip the initial PSW and channel program located - * at address zero and let _text start at 0x200. - */ - _text = 0x200; TEXT_TEXT SCHED_TEXT CPUIDLE_TEXT @@ -47,6 +43,7 @@ SECTIONS KPROBES_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT + *(.text.*_indirect_*) *(.fixup) *(.gnu.warning) } :text = 0x0700 diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index daa09f89ca2d..fcb55b02990e 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -1145,7 +1145,7 @@ void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) * yield-candidate. */ vcpu->preempted = true; - swake_up(&vcpu->wq); + swake_up_one(&vcpu->wq); vcpu->stat.halt_wakeup++; } /* diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 3b7a5151b6a5..f9d90337e64a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -172,6 +172,10 @@ static int nested; module_param(nested, int, S_IRUGO); MODULE_PARM_DESC(nested, "Nested virtualization support"); +/* allow 1m huge page guest backing, if !nested */ +static int hpage; +module_param(hpage, int, 0444); +MODULE_PARM_DESC(hpage, "1m huge page backing support"); /* * For now we handle at most 16 double words as this is what the s390 base @@ -475,6 +479,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_AIS_MIGRATION: r = 1; break; + case KVM_CAP_S390_HPAGE_1M: + r = 0; + if (hpage) + r = 1; + break; case KVM_CAP_S390_MEM_OP: r = MEM_OP_MAX_SIZE; break; @@ -511,19 +520,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) } static void kvm_s390_sync_dirty_log(struct kvm *kvm, - struct kvm_memory_slot *memslot) + struct kvm_memory_slot *memslot) { + int i; gfn_t cur_gfn, last_gfn; - unsigned long address; + unsigned long gaddr, vmaddr; struct gmap *gmap = kvm->arch.gmap; + DECLARE_BITMAP(bitmap, _PAGE_ENTRIES); - /* Loop over all guest pages */ + /* Loop over all guest segments */ + cur_gfn = memslot->base_gfn; last_gfn = memslot->base_gfn + memslot->npages; - for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { - address = gfn_to_hva_memslot(memslot, cur_gfn); + for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) { + gaddr = gfn_to_gpa(cur_gfn); + vmaddr = gfn_to_hva_memslot(memslot, cur_gfn); + if (kvm_is_error_hva(vmaddr)) + continue; + + bitmap_zero(bitmap, _PAGE_ENTRIES); + gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr); + for (i = 0; i < _PAGE_ENTRIES; i++) { + if (test_bit(i, bitmap)) + mark_page_dirty(kvm, cur_gfn + i); + } - if (test_and_clear_guest_dirty(gmap->mm, address)) - mark_page_dirty(kvm, cur_gfn); if (fatal_signal_pending(current)) return; cond_resched(); @@ -667,6 +687,27 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s", r ? "(not available)" : "(success)"); break; + case KVM_CAP_S390_HPAGE_1M: + mutex_lock(&kvm->lock); + if (kvm->created_vcpus) + r = -EBUSY; + else if (!hpage || kvm->arch.use_cmma) + r = -EINVAL; + else { + r = 0; + kvm->mm->context.allow_gmap_hpage_1m = 1; + /* + * We might have to create fake 4k page + * tables. To avoid that the hardware works on + * stale PGSTEs, we emulate these instructions. + */ + kvm->arch.use_skf = 0; + kvm->arch.use_pfmfi = 0; + } + mutex_unlock(&kvm->lock); + VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s", + r ? "(not available)" : "(success)"); + break; case KVM_CAP_S390_USER_STSI: VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); kvm->arch.user_stsi = 1; @@ -714,10 +755,13 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att if (!sclp.has_cmma) break; - ret = -EBUSY; VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support"); mutex_lock(&kvm->lock); - if (!kvm->created_vcpus) { + if (kvm->created_vcpus) + ret = -EBUSY; + else if (kvm->mm->context.allow_gmap_hpage_1m) + ret = -EINVAL; + else { kvm->arch.use_cmma = 1; /* Not compatible with cmma. */ kvm->arch.use_pfmfi = 0; @@ -1540,6 +1584,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) uint8_t *keys; uint64_t hva; int srcu_idx, i, r = 0; + bool unlocked; if (args->flags != 0) return -EINVAL; @@ -1564,9 +1609,11 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) if (r) goto out; + i = 0; down_read(¤t->mm->mmap_sem); srcu_idx = srcu_read_lock(&kvm->srcu); - for (i = 0; i < args->count; i++) { + while (i < args->count) { + unlocked = false; hva = gfn_to_hva(kvm, args->start_gfn + i); if (kvm_is_error_hva(hva)) { r = -EFAULT; @@ -1580,8 +1627,14 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) } r = set_guest_storage_key(current->mm, hva, keys[i], 0); - if (r) - break; + if (r) { + r = fixup_user_fault(current, current->mm, hva, + FAULT_FLAG_WRITE, &unlocked); + if (r) + break; + } + if (!r) + i++; } srcu_read_unlock(&kvm->srcu, srcu_idx); up_read(¤t->mm->mmap_sem); @@ -4082,6 +4135,11 @@ static int __init kvm_s390_init(void) return -ENODEV; } + if (nested && hpage) { + pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently"); + return -EINVAL; + } + for (i = 0; i < 16; i++) kvm_s390_fac_base[i] |= S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i); diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index eb0eb60c7be6..cfc5a62329f6 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -246,9 +246,10 @@ static int try_handle_skey(struct kvm_vcpu *vcpu) static int handle_iske(struct kvm_vcpu *vcpu) { - unsigned long addr; + unsigned long gaddr, vmaddr; unsigned char key; int reg1, reg2; + bool unlocked; int rc; vcpu->stat.instruction_iske++; @@ -262,18 +263,28 @@ static int handle_iske(struct kvm_vcpu *vcpu) kvm_s390_get_regs_rre(vcpu, ®1, ®2); - addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; - addr = kvm_s390_logical_to_effective(vcpu, addr); - addr = kvm_s390_real_to_abs(vcpu, addr); - addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr)); - if (kvm_is_error_hva(addr)) + gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; + gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); + gaddr = kvm_s390_real_to_abs(vcpu, gaddr); + vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); + if (kvm_is_error_hva(vmaddr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - +retry: + unlocked = false; down_read(¤t->mm->mmap_sem); - rc = get_guest_storage_key(current->mm, addr, &key); - up_read(¤t->mm->mmap_sem); + rc = get_guest_storage_key(current->mm, vmaddr, &key); + + if (rc) { + rc = fixup_user_fault(current, current->mm, vmaddr, + FAULT_FLAG_WRITE, &unlocked); + if (!rc) { + up_read(¤t->mm->mmap_sem); + goto retry; + } + } if (rc) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); + up_read(¤t->mm->mmap_sem); vcpu->run->s.regs.gprs[reg1] &= ~0xff; vcpu->run->s.regs.gprs[reg1] |= key; return 0; @@ -281,8 +292,9 @@ static int handle_iske(struct kvm_vcpu *vcpu) static int handle_rrbe(struct kvm_vcpu *vcpu) { - unsigned long addr; + unsigned long vmaddr, gaddr; int reg1, reg2; + bool unlocked; int rc; vcpu->stat.instruction_rrbe++; @@ -296,19 +308,27 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) kvm_s390_get_regs_rre(vcpu, ®1, ®2); - addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; - addr = kvm_s390_logical_to_effective(vcpu, addr); - addr = kvm_s390_real_to_abs(vcpu, addr); - addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr)); - if (kvm_is_error_hva(addr)) + gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; + gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); + gaddr = kvm_s390_real_to_abs(vcpu, gaddr); + vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); + if (kvm_is_error_hva(vmaddr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - +retry: + unlocked = false; down_read(¤t->mm->mmap_sem); - rc = reset_guest_reference_bit(current->mm, addr); - up_read(¤t->mm->mmap_sem); + rc = reset_guest_reference_bit(current->mm, vmaddr); + if (rc < 0) { + rc = fixup_user_fault(current, current->mm, vmaddr, + FAULT_FLAG_WRITE, &unlocked); + if (!rc) { + up_read(¤t->mm->mmap_sem); + goto retry; + } + } if (rc < 0) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - + up_read(¤t->mm->mmap_sem); kvm_s390_set_psw_cc(vcpu, rc); return 0; } @@ -323,6 +343,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) unsigned long start, end; unsigned char key, oldkey; int reg1, reg2; + bool unlocked; int rc; vcpu->stat.instruction_sske++; @@ -355,19 +376,28 @@ static int handle_sske(struct kvm_vcpu *vcpu) } while (start != end) { - unsigned long addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); + unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); + unlocked = false; - if (kvm_is_error_hva(addr)) + if (kvm_is_error_hva(vmaddr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); down_read(¤t->mm->mmap_sem); - rc = cond_set_guest_storage_key(current->mm, addr, key, &oldkey, + rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, m3 & SSKE_NQ, m3 & SSKE_MR, m3 & SSKE_MC); - up_read(¤t->mm->mmap_sem); - if (rc < 0) + + if (rc < 0) { + rc = fixup_user_fault(current, current->mm, vmaddr, + FAULT_FLAG_WRITE, &unlocked); + rc = !rc ? -EAGAIN : rc; + } + if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - start += PAGE_SIZE; + + up_read(¤t->mm->mmap_sem); + if (rc >= 0) + start += PAGE_SIZE; } if (m3 & (SSKE_MC | SSKE_MR)) { @@ -948,15 +978,16 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) } while (start != end) { - unsigned long useraddr; + unsigned long vmaddr; + bool unlocked = false; /* Translate guest address to host address */ - useraddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); - if (kvm_is_error_hva(useraddr)) + vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); + if (kvm_is_error_hva(vmaddr)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { - if (clear_user((void __user *)useraddr, PAGE_SIZE)) + if (clear_user((void __user *)vmaddr, PAGE_SIZE)) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); } @@ -966,14 +997,20 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) if (rc) return rc; down_read(¤t->mm->mmap_sem); - rc = cond_set_guest_storage_key(current->mm, useraddr, + rc = cond_set_guest_storage_key(current->mm, vmaddr, key, NULL, nq, mr, mc); - up_read(¤t->mm->mmap_sem); - if (rc < 0) + if (rc < 0) { + rc = fixup_user_fault(current, current->mm, vmaddr, + FAULT_FLAG_WRITE, &unlocked); + rc = !rc ? -EAGAIN : rc; + } + if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); - } - start += PAGE_SIZE; + up_read(¤t->mm->mmap_sem); + if (rc >= 0) + start += PAGE_SIZE; + } } if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) { diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S index 2311f15be9cf..40c4d59c926e 100644 --- a/arch/s390/lib/mem.S +++ b/arch/s390/lib/mem.S @@ -17,7 +17,7 @@ ENTRY(memmove) ltgr %r4,%r4 lgr %r1,%r2 - bzr %r14 + jz .Lmemmove_exit aghi %r4,-1 clgr %r2,%r3 jnh .Lmemmove_forward @@ -36,6 +36,7 @@ ENTRY(memmove) .Lmemmove_forward_remainder: larl %r5,.Lmemmove_mvc ex %r4,0(%r5) +.Lmemmove_exit: BR_EX %r14 .Lmemmove_reverse: ic %r0,0(%r4,%r3) @@ -65,7 +66,7 @@ EXPORT_SYMBOL(memmove) */ ENTRY(memset) ltgr %r4,%r4 - bzr %r14 + jz .Lmemset_exit ltgr %r3,%r3 jnz .Lmemset_fill aghi %r4,-1 @@ -80,6 +81,7 @@ ENTRY(memset) .Lmemset_clear_remainder: larl %r3,.Lmemset_xc ex %r4,0(%r3) +.Lmemset_exit: BR_EX %r14 .Lmemset_fill: cghi %r4,1 @@ -115,7 +117,7 @@ EXPORT_SYMBOL(memset) */ ENTRY(memcpy) ltgr %r4,%r4 - bzr %r14 + jz .Lmemcpy_exit aghi %r4,-1 srlg %r5,%r4,8 ltgr %r5,%r5 @@ -124,6 +126,7 @@ ENTRY(memcpy) .Lmemcpy_remainder: larl %r5,.Lmemcpy_mvc ex %r4,0(%r5) +.Lmemcpy_exit: BR_EX %r14 .Lmemcpy_loop: mvc 0(256,%r1),0(%r3) @@ -145,9 +148,9 @@ EXPORT_SYMBOL(memcpy) .macro __MEMSET bits,bytes,insn ENTRY(__memset\bits) ltgr %r4,%r4 - bzr %r14 + jz .L__memset_exit\bits cghi %r4,\bytes - je .L__memset_exit\bits + je .L__memset_store\bits aghi %r4,-(\bytes+1) srlg %r5,%r4,8 ltgr %r5,%r5 @@ -163,8 +166,9 @@ ENTRY(__memset\bits) larl %r5,.L__memset_mvc\bits ex %r4,0(%r5) BR_EX %r14 -.L__memset_exit\bits: +.L__memset_store\bits: \insn %r3,0(%r2) +.L__memset_exit\bits: BR_EX %r14 .L__memset_mvc\bits: mvc \bytes(1,%r1),0(%r1) diff --git a/arch/s390/mm/cmm.c b/arch/s390/mm/cmm.c index 6cf024eb2085..510a18299196 100644 --- a/arch/s390/mm/cmm.c +++ b/arch/s390/mm/cmm.c @@ -191,12 +191,7 @@ static void cmm_set_timer(void) del_timer(&cmm_timer); return; } - if (timer_pending(&cmm_timer)) { - if (mod_timer(&cmm_timer, jiffies + cmm_timeout_seconds*HZ)) - return; - } - cmm_timer.expires = jiffies + cmm_timeout_seconds*HZ; - add_timer(&cmm_timer); + mod_timer(&cmm_timer, jiffies + cmm_timeout_seconds * HZ); } static void cmm_timer_fn(struct timer_list *unused) @@ -251,45 +246,42 @@ static int cmm_skip_blanks(char *cp, char **endp) return str != cp; } -static struct ctl_table cmm_table[]; - static int cmm_pages_handler(struct ctl_table *ctl, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { - char buf[16], *p; - unsigned int len; - long nr; + long nr = cmm_get_pages(); + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &nr, + .maxlen = sizeof(long), + }; + int rc; - if (!*lenp || (*ppos && !write)) { - *lenp = 0; - return 0; - } + rc = proc_doulongvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; - if (write) { - len = *lenp; - if (copy_from_user(buf, buffer, - len > sizeof(buf) ? sizeof(buf) : len)) - return -EFAULT; - buf[sizeof(buf) - 1] = '\0'; - cmm_skip_blanks(buf, &p); - nr = simple_strtoul(p, &p, 0); - if (ctl == &cmm_table[0]) - cmm_set_pages(nr); - else - cmm_add_timed_pages(nr); - } else { - if (ctl == &cmm_table[0]) - nr = cmm_get_pages(); - else - nr = cmm_get_timed_pages(); - len = sprintf(buf, "%ld\n", nr); - if (len > *lenp) - len = *lenp; - if (copy_to_user(buffer, buf, len)) - return -EFAULT; - } - *lenp = len; - *ppos += len; + cmm_set_pages(nr); + return 0; +} + +static int cmm_timed_pages_handler(struct ctl_table *ctl, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + long nr = cmm_get_timed_pages(); + struct ctl_table ctl_entry = { + .procname = ctl->procname, + .data = &nr, + .maxlen = sizeof(long), + }; + int rc; + + rc = proc_doulongvec_minmax(&ctl_entry, write, buffer, lenp, ppos); + if (rc < 0 || !write) + return rc; + + cmm_add_timed_pages(nr); return 0; } @@ -338,7 +330,7 @@ static struct ctl_table cmm_table[] = { { .procname = "cmm_timed_pages", .mode = 0644, - .proc_handler = cmm_pages_handler, + .proc_handler = cmm_timed_pages_handler, }, { .procname = "cmm_timeout", diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index 6ad15d3fab81..84111a43ea29 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -80,7 +80,7 @@ struct qin64 { struct dcss_segment { struct list_head list; char dcss_name[8]; - char res_name[15]; + char res_name[16]; unsigned long start_addr; unsigned long end; atomic_t ref_count; @@ -433,7 +433,7 @@ __segment_load (char *name, int do_nonshared, unsigned long *addr, unsigned long memcpy(&seg->res_name, seg->dcss_name, 8); EBCASC(seg->res_name, 8); seg->res_name[8] = '\0'; - strncat(seg->res_name, " (DCSS)", 7); + strlcat(seg->res_name, " (DCSS)", sizeof(seg->res_name)); seg->res->name = seg->res_name; rc = seg->vm_segtype; if (rc == SEG_TYPE_SC || diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e074480d3598..4cc3f06b0ab3 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -502,6 +502,8 @@ retry: /* No reason to continue if interrupted by SIGKILL. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { fault = VM_FAULT_SIGNAL; + if (flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_up; goto out; } if (unlikely(fault & VM_FAULT_ERROR)) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index bc56ec8abcf7..bb44990c8212 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2,8 +2,10 @@ /* * KVM guest address space mapping code * - * Copyright IBM Corp. 2007, 2016 + * Copyright IBM Corp. 2007, 2016, 2018 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * David Hildenbrand <david@redhat.com> + * Janosch Frank <frankja@linux.vnet.ibm.com> */ #include <linux/kernel.h> @@ -521,6 +523,9 @@ void gmap_unlink(struct mm_struct *mm, unsigned long *table, rcu_read_unlock(); } +static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new, + unsigned long gaddr); + /** * gmap_link - set up shadow page tables to connect a host to a guest address * @gmap: pointer to guest mapping meta data structure @@ -541,6 +546,7 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) p4d_t *p4d; pud_t *pud; pmd_t *pmd; + u64 unprot; int rc; BUG_ON(gmap_is_shadow(gmap)); @@ -584,8 +590,8 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) return -EFAULT; pmd = pmd_offset(pud, vmaddr); VM_BUG_ON(pmd_none(*pmd)); - /* large pmds cannot yet be handled */ - if (pmd_large(*pmd)) + /* Are we allowed to use huge pages? */ + if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) return -EFAULT; /* Link gmap segment table entry location to page table. */ rc = radix_tree_preload(GFP_KERNEL); @@ -596,10 +602,22 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) if (*table == _SEGMENT_ENTRY_EMPTY) { rc = radix_tree_insert(&gmap->host_to_guest, vmaddr >> PMD_SHIFT, table); - if (!rc) - *table = pmd_val(*pmd); - } else - rc = 0; + if (!rc) { + if (pmd_large(*pmd)) { + *table = (pmd_val(*pmd) & + _SEGMENT_ENTRY_HARDWARE_BITS_LARGE) + | _SEGMENT_ENTRY_GMAP_UC; + } else + *table = pmd_val(*pmd) & + _SEGMENT_ENTRY_HARDWARE_BITS; + } + } else if (*table & _SEGMENT_ENTRY_PROTECT && + !(pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT)) { + unprot = (u64)*table; + unprot &= ~_SEGMENT_ENTRY_PROTECT; + unprot |= _SEGMENT_ENTRY_GMAP_UC; + gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr); + } spin_unlock(&gmap->guest_table_lock); spin_unlock(ptl); radix_tree_preload_end(); @@ -690,6 +708,12 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) vmaddr |= gaddr & ~PMD_MASK; /* Find vma in the parent mm */ vma = find_vma(gmap->mm, vmaddr); + /* + * We do not discard pages that are backed by + * hugetlbfs, so we don't have to refault them. + */ + if (vma && is_vm_hugetlb_page(vma)) + continue; size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK)); zap_page_range(vma, vmaddr, size); } @@ -864,7 +888,128 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, */ static void gmap_pte_op_end(spinlock_t *ptl) { - spin_unlock(ptl); + if (ptl) + spin_unlock(ptl); +} + +/** + * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock + * and return the pmd pointer + * @gmap: pointer to guest mapping meta data structure + * @gaddr: virtual address in the guest address space + * + * Returns a pointer to the pmd for a guest address, or NULL + */ +static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr) +{ + pmd_t *pmdp; + + BUG_ON(gmap_is_shadow(gmap)); + spin_lock(&gmap->guest_table_lock); + pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); + + if (!pmdp || pmd_none(*pmdp)) { + spin_unlock(&gmap->guest_table_lock); + return NULL; + } + + /* 4k page table entries are locked via the pte (pte_alloc_map_lock). */ + if (!pmd_large(*pmdp)) + spin_unlock(&gmap->guest_table_lock); + return pmdp; +} + +/** + * gmap_pmd_op_end - release the guest_table_lock if needed + * @gmap: pointer to the guest mapping meta data structure + * @pmdp: pointer to the pmd + */ +static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) +{ + if (pmd_large(*pmdp)) + spin_unlock(&gmap->guest_table_lock); +} + +/* + * gmap_protect_pmd - remove access rights to memory and set pmd notification bits + * @pmdp: pointer to the pmd to be protected + * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE + * @bits: notification bits to set + * + * Returns: + * 0 if successfully protected + * -EAGAIN if a fixup is needed + * -EINVAL if unsupported notifier bits have been specified + * + * Expected to be called with sg->mm->mmap_sem in read and + * guest_table_lock held. + */ +static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, + pmd_t *pmdp, int prot, unsigned long bits) +{ + int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID; + int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT; + pmd_t new = *pmdp; + + /* Fixup needed */ + if ((pmd_i && (prot != PROT_NONE)) || (pmd_p && (prot == PROT_WRITE))) + return -EAGAIN; + + if (prot == PROT_NONE && !pmd_i) { + pmd_val(new) |= _SEGMENT_ENTRY_INVALID; + gmap_pmdp_xchg(gmap, pmdp, new, gaddr); + } + + if (prot == PROT_READ && !pmd_p) { + pmd_val(new) &= ~_SEGMENT_ENTRY_INVALID; + pmd_val(new) |= _SEGMENT_ENTRY_PROTECT; + gmap_pmdp_xchg(gmap, pmdp, new, gaddr); + } + + if (bits & GMAP_NOTIFY_MPROT) + pmd_val(*pmdp) |= _SEGMENT_ENTRY_GMAP_IN; + + /* Shadow GMAP protection needs split PMDs */ + if (bits & GMAP_NOTIFY_SHADOW) + return -EINVAL; + + return 0; +} + +/* + * gmap_protect_pte - remove access rights to memory and set pgste bits + * @gmap: pointer to guest mapping meta data structure + * @gaddr: virtual address in the guest address space + * @pmdp: pointer to the pmd associated with the pte + * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE + * @bits: notification bits to set + * + * Returns 0 if successfully protected, -ENOMEM if out of memory and + * -EAGAIN if a fixup is needed. + * + * Expected to be called with sg->mm->mmap_sem in read + */ +static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, + pmd_t *pmdp, int prot, unsigned long bits) +{ + int rc; + pte_t *ptep; + spinlock_t *ptl = NULL; + unsigned long pbits = 0; + + if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) + return -EAGAIN; + + ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); + if (!ptep) + return -ENOMEM; + + pbits |= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0; + pbits |= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0; + /* Protect and unlock. */ + rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); + gmap_pte_op_end(ptl); + return rc; } /* @@ -883,30 +1028,45 @@ static void gmap_pte_op_end(spinlock_t *ptl) static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, unsigned long len, int prot, unsigned long bits) { - unsigned long vmaddr; - spinlock_t *ptl; - pte_t *ptep; + unsigned long vmaddr, dist; + pmd_t *pmdp; int rc; BUG_ON(gmap_is_shadow(gmap)); while (len) { rc = -EAGAIN; - ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); - if (ptep) { - rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, bits); - gmap_pte_op_end(ptl); + pmdp = gmap_pmd_op_walk(gmap, gaddr); + if (pmdp) { + if (!pmd_large(*pmdp)) { + rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, + bits); + if (!rc) { + len -= PAGE_SIZE; + gaddr += PAGE_SIZE; + } + } else { + rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, + bits); + if (!rc) { + dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK); + len = len < dist ? 0 : len - dist; + gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE; + } + } + gmap_pmd_op_end(gmap, pmdp); } if (rc) { + if (rc == -EINVAL) + return rc; + + /* -EAGAIN, fixup of userspace mm and gmap */ vmaddr = __gmap_translate(gmap, gaddr); if (IS_ERR_VALUE(vmaddr)) return vmaddr; rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot); if (rc) return rc; - continue; } - gaddr += PAGE_SIZE; - len -= PAGE_SIZE; } return 0; } @@ -935,7 +1095,7 @@ int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, if (!MACHINE_HAS_ESOP && prot == PROT_READ) return -EINVAL; down_read(&gmap->mm->mmap_sem); - rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT); + rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT); up_read(&gmap->mm->mmap_sem); return rc; } @@ -1474,6 +1634,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, unsigned long limit; int rc; + BUG_ON(parent->mm->context.allow_gmap_hpage_1m); BUG_ON(gmap_is_shadow(parent)); spin_lock(&parent->shadow_lock); sg = gmap_find_shadow(parent, asce, edat_level); @@ -1526,7 +1687,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, down_read(&parent->mm->mmap_sem); rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN, ((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE, - PROT_READ, PGSTE_VSIE_BIT); + PROT_READ, GMAP_NOTIFY_SHADOW); up_read(&parent->mm->mmap_sem); spin_lock(&parent->shadow_lock); new->initialized = true; @@ -2092,6 +2253,225 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr, } EXPORT_SYMBOL_GPL(ptep_notify); +static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp, + unsigned long gaddr) +{ + pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_IN; + gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); +} + +/** + * gmap_pmdp_xchg - exchange a gmap pmd with another + * @gmap: pointer to the guest address space structure + * @pmdp: pointer to the pmd entry + * @new: replacement entry + * @gaddr: the affected guest address + * + * This function is assumed to be called with the guest_table_lock + * held. + */ +static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, + unsigned long gaddr) +{ + gaddr &= HPAGE_MASK; + pmdp_notify_gmap(gmap, pmdp, gaddr); + pmd_val(new) &= ~_SEGMENT_ENTRY_GMAP_IN; + if (MACHINE_HAS_TLB_GUEST) + __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, + IDTE_GLOBAL); + else if (MACHINE_HAS_IDTE) + __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL); + else + __pmdp_csp(pmdp); + *pmdp = new; +} + +static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr, + int purge) +{ + pmd_t *pmdp; + struct gmap *gmap; + unsigned long gaddr; + + rcu_read_lock(); + list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { + spin_lock(&gmap->guest_table_lock); + pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest, + vmaddr >> PMD_SHIFT); + if (pmdp) { + gaddr = __gmap_segment_gaddr((unsigned long *)pmdp); + pmdp_notify_gmap(gmap, pmdp, gaddr); + WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | + _SEGMENT_ENTRY_GMAP_UC)); + if (purge) + __pmdp_csp(pmdp); + pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; + } + spin_unlock(&gmap->guest_table_lock); + } + rcu_read_unlock(); +} + +/** + * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without + * flushing + * @mm: pointer to the process mm_struct + * @vmaddr: virtual address in the process address space + */ +void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr) +{ + gmap_pmdp_clear(mm, vmaddr, 0); +} +EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate); + +/** + * gmap_pmdp_csp - csp all affected guest pmd entries + * @mm: pointer to the process mm_struct + * @vmaddr: virtual address in the process address space + */ +void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr) +{ + gmap_pmdp_clear(mm, vmaddr, 1); +} +EXPORT_SYMBOL_GPL(gmap_pmdp_csp); + +/** + * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry + * @mm: pointer to the process mm_struct + * @vmaddr: virtual address in the process address space + */ +void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr) +{ + unsigned long *entry, gaddr; + struct gmap *gmap; + pmd_t *pmdp; + + rcu_read_lock(); + list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { + spin_lock(&gmap->guest_table_lock); + entry = radix_tree_delete(&gmap->host_to_guest, + vmaddr >> PMD_SHIFT); + if (entry) { + pmdp = (pmd_t *)entry; + gaddr = __gmap_segment_gaddr(entry); + pmdp_notify_gmap(gmap, pmdp, gaddr); + WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | + _SEGMENT_ENTRY_GMAP_UC)); + if (MACHINE_HAS_TLB_GUEST) + __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, + gmap->asce, IDTE_LOCAL); + else if (MACHINE_HAS_IDTE) + __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL); + *entry = _SEGMENT_ENTRY_EMPTY; + } + spin_unlock(&gmap->guest_table_lock); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local); + +/** + * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry + * @mm: pointer to the process mm_struct + * @vmaddr: virtual address in the process address space + */ +void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr) +{ + unsigned long *entry, gaddr; + struct gmap *gmap; + pmd_t *pmdp; + + rcu_read_lock(); + list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { + spin_lock(&gmap->guest_table_lock); + entry = radix_tree_delete(&gmap->host_to_guest, + vmaddr >> PMD_SHIFT); + if (entry) { + pmdp = (pmd_t *)entry; + gaddr = __gmap_segment_gaddr(entry); + pmdp_notify_gmap(gmap, pmdp, gaddr); + WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE | + _SEGMENT_ENTRY_GMAP_UC)); + if (MACHINE_HAS_TLB_GUEST) + __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, + gmap->asce, IDTE_GLOBAL); + else if (MACHINE_HAS_IDTE) + __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL); + else + __pmdp_csp(pmdp); + *entry = _SEGMENT_ENTRY_EMPTY; + } + spin_unlock(&gmap->guest_table_lock); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global); + +/** + * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status + * @gmap: pointer to guest address space + * @pmdp: pointer to the pmd to be tested + * @gaddr: virtual address in the guest address space + * + * This function is assumed to be called with the guest_table_lock + * held. + */ +bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp, + unsigned long gaddr) +{ + if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID) + return false; + + /* Already protected memory, which did not change is clean */ + if (pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT && + !(pmd_val(*pmdp) & _SEGMENT_ENTRY_GMAP_UC)) + return false; + + /* Clear UC indication and reset protection */ + pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_UC; + gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0); + return true; +} + +/** + * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment + * @gmap: pointer to guest address space + * @bitmap: dirty bitmap for this pmd + * @gaddr: virtual address in the guest address space + * @vmaddr: virtual address in the host address space + * + * This function is assumed to be called with the guest_table_lock + * held. + */ +void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4], + unsigned long gaddr, unsigned long vmaddr) +{ + int i; + pmd_t *pmdp; + pte_t *ptep; + spinlock_t *ptl; + + pmdp = gmap_pmd_op_walk(gmap, gaddr); + if (!pmdp) + return; + + if (pmd_large(*pmdp)) { + if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr)) + bitmap_fill(bitmap, _PAGE_ENTRIES); + } else { + for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) { + ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); + if (!ptep) + continue; + if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) + set_bit(i, bitmap); + spin_unlock(ptl); + } + } + gmap_pmd_op_end(gmap, pmdp); +} +EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd); + static inline void thp_split_mm(struct mm_struct *mm) { #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -2168,17 +2548,45 @@ EXPORT_SYMBOL_GPL(s390_enable_sie); * Enable storage key handling from now on and initialize the storage * keys with the default key. */ -static int __s390_enable_skey(pte_t *pte, unsigned long addr, - unsigned long next, struct mm_walk *walk) +static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr, + unsigned long next, struct mm_walk *walk) { /* Clear storage key */ ptep_zap_key(walk->mm, addr, pte); return 0; } +static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, + unsigned long hmask, unsigned long next, + struct mm_walk *walk) +{ + pmd_t *pmd = (pmd_t *)pte; + unsigned long start, end; + struct page *page = pmd_page(*pmd); + + /* + * The write check makes sure we do not set a key on shared + * memory. This is needed as the walker does not differentiate + * between actual guest memory and the process executable or + * shared libraries. + */ + if (pmd_val(*pmd) & _SEGMENT_ENTRY_INVALID || + !(pmd_val(*pmd) & _SEGMENT_ENTRY_WRITE)) + return 0; + + start = pmd_val(*pmd) & HPAGE_MASK; + end = start + HPAGE_SIZE - 1; + __storage_key_init_range(start, end); + set_bit(PG_arch_1, &page->flags); + return 0; +} + int s390_enable_skey(void) { - struct mm_walk walk = { .pte_entry = __s390_enable_skey }; + struct mm_walk walk = { + .hugetlb_entry = __s390_enable_skey_hugetlb, + .pte_entry = __s390_enable_skey_pte, + }; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int rc = 0; diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index e804090f4470..b0246c705a19 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -123,6 +123,29 @@ static inline pte_t __rste_to_pte(unsigned long rste) return pte; } +static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste) +{ + struct page *page; + unsigned long size, paddr; + + if (!mm_uses_skeys(mm) || + rste & _SEGMENT_ENTRY_INVALID) + return; + + if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) { + page = pud_page(__pud(rste)); + size = PUD_SIZE; + paddr = rste & PUD_MASK; + } else { + page = pmd_page(__pmd(rste)); + size = PMD_SIZE; + paddr = rste & PMD_MASK; + } + + if (!test_and_set_bit(PG_arch_1, &page->flags)) + __storage_key_init_range(paddr, paddr + size - 1); +} + void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { @@ -137,6 +160,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE; else rste |= _SEGMENT_ENTRY_LARGE; + clear_huge_pte_skeys(mm, rste); pte_val(*ptep) = rste; } diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 382153ff17e3..dc3cede7f2ec 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -271,7 +271,7 @@ void arch_set_page_states(int make_stable) list_for_each(l, &zone->free_area[order].free_list[t]) { page = list_entry(l, struct page, lru); if (make_stable) - set_page_stable_dat(page, 0); + set_page_stable_dat(page, order); else set_page_unused(page, order); } diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index c44171588d08..f8c6faab41f4 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -14,7 +14,7 @@ static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) { - asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0" + asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0" : [addr] "+a" (addr) : [skey] "d" (skey)); return addr; } @@ -23,8 +23,6 @@ void __storage_key_init_range(unsigned long start, unsigned long end) { unsigned long boundary, size; - if (!PAGE_DEFAULT_KEY) - return; while (start < end) { if (MACHINE_HAS_EDAT1) { /* set storage keys for a 1MB frame */ @@ -37,7 +35,7 @@ void __storage_key_init_range(unsigned long start, unsigned long end) continue; } } - page_set_storage_key(start, PAGE_DEFAULT_KEY, 0); + page_set_storage_key(start, PAGE_DEFAULT_KEY, 1); start += PAGE_SIZE; } } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 84bd6329a88d..76d89ee8b428 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -28,7 +28,7 @@ static struct ctl_table page_table_sysctl[] = { .data = &page_table_allocate_pgste, .maxlen = sizeof(int), .mode = S_IRUGO | S_IWUSR, - .proc_handler = proc_dointvec, + .proc_handler = proc_dointvec_minmax, .extra1 = &page_table_allocate_pgste_min, .extra2 = &page_table_allocate_pgste_max, }, @@ -252,6 +252,8 @@ void page_table_free(struct mm_struct *mm, unsigned long *table) spin_unlock_bh(&mm->context.lock); if (mask != 0) return; + } else { + atomic_xor_bits(&page->_refcount, 3U << 24); } pgtable_page_dtor(page); @@ -304,6 +306,8 @@ static void __tlb_remove_table(void *_table) break; /* fallthrough */ case 3: /* 4K page table with pgstes */ + if (mask & 3) + atomic_xor_bits(&page->_refcount, 3 << 24); pgtable_page_dtor(page); __free_page(page); break; diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 301e466e4263..f2cc7da473e4 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -347,18 +347,27 @@ static inline void pmdp_idte_local(struct mm_struct *mm, mm->context.asce, IDTE_LOCAL); else __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); + if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) + gmap_pmdp_idte_local(mm, addr); } static inline void pmdp_idte_global(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - if (MACHINE_HAS_TLB_GUEST) + if (MACHINE_HAS_TLB_GUEST) { __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_GLOBAL); - else if (MACHINE_HAS_IDTE) + if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) + gmap_pmdp_idte_global(mm, addr); + } else if (MACHINE_HAS_IDTE) { __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); - else + if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) + gmap_pmdp_idte_global(mm, addr); + } else { __pmdp_csp(pmdp); + if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) + gmap_pmdp_csp(mm, addr); + } } static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, @@ -392,6 +401,8 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, cpumask_of(smp_processor_id()))) { pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; mm->context.flush_mm = 1; + if (mm_has_pgste(mm)) + gmap_pmdp_invalidate(mm, addr); } else { pmdp_idte_global(mm, addr, pmdp); } @@ -399,6 +410,24 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, return old; } +static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + if (!p4d) + return NULL; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(mm, pud, addr); + return pmd; +} + pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t new) { @@ -693,40 +722,14 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) /* * Test and reset if a guest page is dirty */ -bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) +bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { - spinlock_t *ptl; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; pgste_t pgste; - pte_t *ptep; pte_t pte; bool dirty; int nodat; - pgd = pgd_offset(mm, addr); - p4d = p4d_alloc(mm, pgd, addr); - if (!p4d) - return false; - pud = pud_alloc(mm, p4d, addr); - if (!pud) - return false; - pmd = pmd_alloc(mm, pud, addr); - if (!pmd) - return false; - /* We can't run guests backed by huge pages, but userspace can - * still set them up and then try to migrate them without any - * migration support. - */ - if (pmd_large(*pmd)) - return true; - - ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl); - if (unlikely(!ptep)) - return false; - pgste = pgste_get_lock(ptep); dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); pgste_val(pgste) &= ~PGSTE_UC_BIT; @@ -742,21 +745,43 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) *ptep = pte; } pgste_set_unlock(ptep, pgste); - - spin_unlock(ptl); return dirty; } -EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); +EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc); int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, unsigned char key, bool nq) { - unsigned long keyul; + unsigned long keyul, paddr; spinlock_t *ptl; pgste_t old, new; + pmd_t *pmdp; pte_t *ptep; - ptep = get_locked_pte(mm, addr, &ptl); + pmdp = pmd_alloc_map(mm, addr); + if (unlikely(!pmdp)) + return -EFAULT; + + ptl = pmd_lock(mm, pmdp); + if (!pmd_present(*pmdp)) { + spin_unlock(ptl); + return -EFAULT; + } + + if (pmd_large(*pmdp)) { + paddr = pmd_val(*pmdp) & HPAGE_MASK; + paddr |= addr & ~HPAGE_MASK; + /* + * Huge pmds need quiescing operations, they are + * always mapped. + */ + page_set_storage_key(paddr, key, 1); + spin_unlock(ptl); + return 0; + } + spin_unlock(ptl); + + ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); if (unlikely(!ptep)) return -EFAULT; @@ -767,14 +792,14 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; if (!(pte_val(*ptep) & _PAGE_INVALID)) { - unsigned long address, bits, skey; + unsigned long bits, skey; - address = pte_val(*ptep) & PAGE_MASK; - skey = (unsigned long) page_get_storage_key(address); + paddr = pte_val(*ptep) & PAGE_MASK; + skey = (unsigned long) page_get_storage_key(paddr); bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); /* Set storage key ACC and FP */ - page_set_storage_key(address, skey, !nq); + page_set_storage_key(paddr, skey, !nq); /* Merge host changed & referenced into pgste */ pgste_val(new) |= bits << 52; } @@ -830,11 +855,32 @@ EXPORT_SYMBOL(cond_set_guest_storage_key); int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) { spinlock_t *ptl; + unsigned long paddr; pgste_t old, new; + pmd_t *pmdp; pte_t *ptep; int cc = 0; - ptep = get_locked_pte(mm, addr, &ptl); + pmdp = pmd_alloc_map(mm, addr); + if (unlikely(!pmdp)) + return -EFAULT; + + ptl = pmd_lock(mm, pmdp); + if (!pmd_present(*pmdp)) { + spin_unlock(ptl); + return -EFAULT; + } + + if (pmd_large(*pmdp)) { + paddr = pmd_val(*pmdp) & HPAGE_MASK; + paddr |= addr & ~HPAGE_MASK; + cc = page_reset_referenced(paddr); + spin_unlock(ptl); + return cc; + } + spin_unlock(ptl); + + ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); if (unlikely(!ptep)) return -EFAULT; @@ -843,7 +889,8 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) pgste_val(new) &= ~PGSTE_GR_BIT; if (!(pte_val(*ptep) & _PAGE_INVALID)) { - cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); + paddr = pte_val(*ptep) & PAGE_MASK; + cc = page_reset_referenced(paddr); /* Merge real referenced bit into host-set */ pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; } @@ -862,18 +909,42 @@ EXPORT_SYMBOL(reset_guest_reference_bit); int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, unsigned char *key) { + unsigned long paddr; spinlock_t *ptl; pgste_t pgste; + pmd_t *pmdp; pte_t *ptep; - ptep = get_locked_pte(mm, addr, &ptl); + pmdp = pmd_alloc_map(mm, addr); + if (unlikely(!pmdp)) + return -EFAULT; + + ptl = pmd_lock(mm, pmdp); + if (!pmd_present(*pmdp)) { + /* Not yet mapped memory has a zero key */ + spin_unlock(ptl); + *key = 0; + return 0; + } + + if (pmd_large(*pmdp)) { + paddr = pmd_val(*pmdp) & HPAGE_MASK; + paddr |= addr & ~HPAGE_MASK; + *key = page_get_storage_key(paddr); + spin_unlock(ptl); + return 0; + } + spin_unlock(ptl); + + ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); if (unlikely(!ptep)) return -EFAULT; pgste = pgste_get_lock(ptep); *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; + paddr = pte_val(*ptep) & PAGE_MASK; if (!(pte_val(*ptep) & _PAGE_INVALID)) - *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); + *key = page_get_storage_key(paddr); /* Reflect guest's logical view, not physical */ *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; pgste_set_unlock(ptep, pgste); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index d2db8acb1a55..d7052cbe984f 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -485,8 +485,6 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) /* br %r1 */ _EMIT2(0x07f1); } else { - /* larl %r1,.+14 */ - EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14); /* ex 0,S390_lowcore.br_r1_tampoline */ EMIT4_DISP(0x44000000, REG_0, REG_0, offsetof(struct lowcore, br_r1_trampoline)); @@ -1286,6 +1284,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) goto free_addrs; } if (bpf_jit_prog(&jit, fp)) { + bpf_jit_binary_free(header); fp = orig_fp; goto free_addrs; } diff --git a/arch/s390/numa/numa.c b/arch/s390/numa/numa.c index 06a80434cfe6..5bd374491f94 100644 --- a/arch/s390/numa/numa.c +++ b/arch/s390/numa/numa.c @@ -134,6 +134,8 @@ void __init numa_setup(void) { pr_info("NUMA mode: %s\n", mode->name); nodes_clear(node_possible_map); + /* Initially attach all possible CPUs to node 0. */ + cpumask_copy(&node_to_cpumask_map[0], cpu_possible_mask); if (mode->setup) mode->setup(); numa_setup_memory(); @@ -141,20 +143,6 @@ void __init numa_setup(void) } /* - * numa_init_early() - Initialization initcall - * - * This runs when only one CPU is online and before the first - * topology update is called for by the scheduler. - */ -static int __init numa_init_early(void) -{ - /* Attach all possible CPUs to node 0 for now. */ - cpumask_copy(&node_to_cpumask_map[0], cpu_possible_mask); - return 0; -} -early_initcall(numa_init_early); - -/* * numa_init_late() - Initialization initcall * * Register NUMA nodes. diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index b482e95b6249..57f7cdac70a3 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -48,6 +48,10 @@ static char *pci_fmt2_names[] = { "Maximum work units", }; +static char *pci_fmt3_names[] = { + "Transmitted bytes", +}; + static char *pci_sw_names[] = { "Allocated pages", "Mapped pages", @@ -112,6 +116,10 @@ static int pci_perf_show(struct seq_file *m, void *v) pci_fmb_show(m, pci_fmt2_names, ARRAY_SIZE(pci_fmt2_names), &zdev->fmb->fmt2.consumed_work_units); break; + case 3: + pci_fmb_show(m, pci_fmt3_names, ARRAY_SIZE(pci_fmt3_names), + &zdev->fmb->fmt3.tx_bytes); + break; default: seq_puts(m, "Unknown format\n"); } diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 1ace023cbdce..8d61218a71aa 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -7,13 +7,13 @@ purgatory-y := head.o purgatory.o string.o sha256.o mem.o targets += $(purgatory-y) purgatory.ro kexec-purgatory.c PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) -$(obj)/sha256.o: $(srctree)/lib/sha256.c +$(obj)/sha256.o: $(srctree)/lib/sha256.c FORCE $(call if_changed_rule,cc_o_c) -$(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S +$(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE $(call if_changed_rule,as_o_S) -$(obj)/string.o: $(srctree)/arch/s390/lib/string.c +$(obj)/string.o: $(srctree)/arch/s390/lib/string.c FORCE $(call if_changed_rule,cc_o_c) LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined -nostdlib @@ -21,8 +21,9 @@ LDFLAGS_purgatory.ro += -z nodefaultlib KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding -KBUILD_CFLAGS += -c -MD -Os -m64 -msoft-float +KBUILD_CFLAGS += -c -MD -Os -m64 -msoft-float -fno-common KBUILD_CFLAGS += $(call cc-option,-fno-PIE) +KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld) diff --git a/arch/s390/purgatory/head.S b/arch/s390/purgatory/head.S index 660c96a05a9b..2e3707b12edd 100644 --- a/arch/s390/purgatory/head.S +++ b/arch/s390/purgatory/head.S @@ -243,33 +243,26 @@ gprregs: .quad 0 .endr -purgatory_sha256_digest: - .global purgatory_sha256_digest - .rept 32 /* SHA256_DIGEST_SIZE */ - .byte 0 - .endr - -purgatory_sha_regions: - .global purgatory_sha_regions - .rept 16 * __KEXEC_SHA_REGION_SIZE /* KEXEC_SEGMENTS_MAX */ - .byte 0 - .endr - -kernel_entry: - .global kernel_entry - .quad 0 - -kernel_type: - .global kernel_type - .quad 0 - -crash_start: - .global crash_start - .quad 0 +/* Macro to define a global variable with name and size (in bytes) to be + * shared with C code. + * + * Add the .size and .type attribute to satisfy checks on the Elf_Sym during + * purgatory load. + */ +.macro GLOBAL_VARIABLE name,size +\name: + .global \name + .size \name,\size + .type \name,object + .skip \size,0 +.endm -crash_size: - .global crash_size - .quad 0 +GLOBAL_VARIABLE purgatory_sha256_digest,32 +GLOBAL_VARIABLE purgatory_sha_regions,16*__KEXEC_SHA_REGION_SIZE +GLOBAL_VARIABLE kernel_entry,8 +GLOBAL_VARIABLE kernel_type,8 +GLOBAL_VARIABLE crash_start,8 +GLOBAL_VARIABLE crash_size,8 .align PAGE_SIZE stack: diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 4e2beb3c29b7..3528e6da4e87 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -12,15 +12,6 @@ #include <linux/string.h> #include <asm/purgatory.h> -struct kexec_sha_region purgatory_sha_regions[KEXEC_SEGMENT_MAX]; -u8 purgatory_sha256_digest[SHA256_DIGEST_SIZE]; - -u64 kernel_entry; -u64 kernel_type; - -u64 crash_start; -u64 crash_size; - int verify_sha256_digest(void) { struct kexec_sha_region *ptr, *end; diff --git a/arch/s390/scripts/Makefile.chkbss b/arch/s390/scripts/Makefile.chkbss index d92f2d94a5d9..9bba2c14e0ca 100644 --- a/arch/s390/scripts/Makefile.chkbss +++ b/arch/s390/scripts/Makefile.chkbss @@ -2,13 +2,22 @@ quiet_cmd_chkbss = CHKBSS $< define cmd_chkbss + rm -f $@; \ if ! $(OBJDUMP) -j .bss -w -h $< | awk 'END { if ($$3) exit 1 }'; then \ echo "error: $< .bss section is not empty" >&2; exit 1; \ fi; \ touch $@; endef -$(obj)/built-in.a: $(patsubst %, $(obj)/%.chkbss, $(chkbss)) +chkbss-target ?= $(obj)/built-in.a +ifneq (,$(findstring /,$(chkbss))) +chkbss-files := $(patsubst %, %.chkbss, $(chkbss)) +else +chkbss-files := $(patsubst %, $(obj)/%.chkbss, $(chkbss)) +endif + +$(chkbss-target): $(chkbss-files) +targets += $(notdir $(chkbss-files)) %.o.chkbss: %.o $(call cmd,chkbss) diff --git a/arch/s390/tools/gen_opcode_table.c b/arch/s390/tools/gen_opcode_table.c index 259aa0680d1a..a1bc02b29c81 100644 --- a/arch/s390/tools/gen_opcode_table.c +++ b/arch/s390/tools/gen_opcode_table.c @@ -257,7 +257,7 @@ static void add_to_group(struct gen_opcode *desc, struct insn *insn, int offset) if (!desc->group) exit(EXIT_FAILURE); group = &desc->group[desc->nr_groups - 1]; - strncpy(group->opcode, insn->opcode, 2); + memcpy(group->opcode, insn->opcode, 2); group->type = insn->type; group->offset = offset; group->count = 1; @@ -283,7 +283,7 @@ static void print_opcode_table(struct gen_opcode *desc) continue; add_to_group(desc, insn, offset); if (strncmp(opcode, insn->opcode, 2)) { - strncpy(opcode, insn->opcode, 2); + memcpy(opcode, insn->opcode, 2); printf("\t/* %.2s */ \\\n", opcode); } print_opcode(insn, offset); diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index 0fd0099f43cc..f37b95a80232 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -32,44 +32,9 @@ #include <asm/atomic-irq.h> #endif -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) -#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) - -#define atomic_inc(v) atomic_add(1, (v)) -#define atomic_dec(v) atomic_sub(1, (v)) - #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -/** - * __atomic_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - - return c; -} - #endif /* CONFIG_CPU_J2 */ #endif /* __ASM_SH_ATOMIC_H */ diff --git a/arch/sh/include/asm/cmpxchg-xchg.h b/arch/sh/include/asm/cmpxchg-xchg.h index 1e881f5db659..593a9704782b 100644 --- a/arch/sh/include/asm/cmpxchg-xchg.h +++ b/arch/sh/include/asm/cmpxchg-xchg.h @@ -8,7 +8,8 @@ * This work is licensed under the terms of the GNU GPL, version 2. See the * file "COPYING" in the main directory of this archive for more details. */ -#include <linux/bitops.h> +#include <linux/bits.h> +#include <linux/compiler.h> #include <asm/byteorder.h> /* diff --git a/arch/sh/include/asm/hw_breakpoint.h b/arch/sh/include/asm/hw_breakpoint.h index 7431c172c0cb..199d17b765f2 100644 --- a/arch/sh/include/asm/hw_breakpoint.h +++ b/arch/sh/include/asm/hw_breakpoint.h @@ -10,7 +10,6 @@ #include <linux/types.h> struct arch_hw_breakpoint { - char *name; /* Contains name of the symbol to set bkpt */ unsigned long address; u16 len; u16 type; @@ -41,6 +40,7 @@ struct sh_ubc { struct clk *clk; /* optional interface clock / MSTP bit */ }; +struct perf_event_attr; struct perf_event; struct task_struct; struct pmu; @@ -54,8 +54,10 @@ static inline int hw_breakpoint_slots(int type) } /* arch/sh/kernel/hw_breakpoint.c */ -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); diff --git a/arch/sh/include/asm/kprobes.h b/arch/sh/include/asm/kprobes.h index 85d8bcaa8493..6171682f7798 100644 --- a/arch/sh/include/asm/kprobes.h +++ b/arch/sh/include/asm/kprobes.h @@ -27,7 +27,6 @@ struct kprobe; void arch_remove_kprobe(struct kprobe *); void kretprobe_trampoline(void); -void jprobe_return_end(void); /* Architecture specific copy of original instruction*/ struct arch_specific_insn { @@ -43,9 +42,6 @@ struct prev_kprobe { /* per-cpu kprobe control block */ struct kprobe_ctlblk { unsigned long kprobe_status; - unsigned long jprobe_saved_r15; - struct pt_regs jprobe_saved_regs; - kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/sh/kernel/hw_breakpoint.c b/arch/sh/kernel/hw_breakpoint.c index 8648ed05ccf0..d9ff3b42da7c 100644 --- a/arch/sh/kernel/hw_breakpoint.c +++ b/arch/sh/kernel/hw_breakpoint.c @@ -124,14 +124,13 @@ static int get_hbp_len(u16 hbp_len) /* * Check for virtual address in kernel space. */ -int arch_check_bp_in_kernelspace(struct perf_event *bp) +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { unsigned int len; unsigned long va; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - va = info->address; - len = get_hbp_len(info->len); + va = hw->address; + len = get_hbp_len(hw->len); return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); } @@ -174,40 +173,40 @@ int arch_bp_generic_fields(int sh_len, int sh_type, return 0; } -static int arch_build_bp_info(struct perf_event *bp) +static int arch_build_bp_info(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - - info->address = bp->attr.bp_addr; + hw->address = attr->bp_addr; /* Len */ - switch (bp->attr.bp_len) { + switch (attr->bp_len) { case HW_BREAKPOINT_LEN_1: - info->len = SH_BREAKPOINT_LEN_1; + hw->len = SH_BREAKPOINT_LEN_1; break; case HW_BREAKPOINT_LEN_2: - info->len = SH_BREAKPOINT_LEN_2; + hw->len = SH_BREAKPOINT_LEN_2; break; case HW_BREAKPOINT_LEN_4: - info->len = SH_BREAKPOINT_LEN_4; + hw->len = SH_BREAKPOINT_LEN_4; break; case HW_BREAKPOINT_LEN_8: - info->len = SH_BREAKPOINT_LEN_8; + hw->len = SH_BREAKPOINT_LEN_8; break; default: return -EINVAL; } /* Type */ - switch (bp->attr.bp_type) { + switch (attr->bp_type) { case HW_BREAKPOINT_R: - info->type = SH_BREAKPOINT_READ; + hw->type = SH_BREAKPOINT_READ; break; case HW_BREAKPOINT_W: - info->type = SH_BREAKPOINT_WRITE; + hw->type = SH_BREAKPOINT_WRITE; break; case HW_BREAKPOINT_W | HW_BREAKPOINT_R: - info->type = SH_BREAKPOINT_RW; + hw->type = SH_BREAKPOINT_RW; break; default: return -EINVAL; @@ -219,19 +218,20 @@ static int arch_build_bp_info(struct perf_event *bp) /* * Validate the arch-specific HW Breakpoint register settings */ -int arch_validate_hwbkpt_settings(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); unsigned int align; int ret; - ret = arch_build_bp_info(bp); + ret = arch_build_bp_info(bp, attr, hw); if (ret) return ret; ret = -EINVAL; - switch (info->len) { + switch (hw->len) { case SH_BREAKPOINT_LEN_1: align = 0; break; @@ -249,17 +249,10 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) } /* - * For kernel-addresses, either the address or symbol name can be - * specified. - */ - if (info->name) - info->address = (unsigned long)kallsyms_lookup_name(info->name); - - /* * Check that the low-order bits of the address are appropriate * for the alignment implied by len. */ - if (info->address & align) + if (hw->address & align) return -EINVAL; return 0; @@ -346,7 +339,7 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args) perf_bp_event(bp, args->regs); /* Deliver the signal to userspace */ - if (!arch_check_bp_in_kernelspace(bp)) { + if (!arch_check_bp_in_kernelspace(&bp->hw.info)) { force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)NULL, current); } diff --git a/arch/sh/kernel/kprobes.c b/arch/sh/kernel/kprobes.c index 52a5e11247d1..241e903dd3ee 100644 --- a/arch/sh/kernel/kprobes.c +++ b/arch/sh/kernel/kprobes.c @@ -248,11 +248,6 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) prepare_singlestep(p, regs); kcb->kprobe_status = KPROBE_REENTER; return 1; - } else { - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - goto ss_probe; - } } goto no_kprobe; } @@ -277,11 +272,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) set_current_kprobe(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (p->pre_handler && p->pre_handler(p, regs)) + if (p->pre_handler && p->pre_handler(p, regs)) { /* handler has already set things up, so skip ss setup */ + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; + } -ss_probe: prepare_singlestep(p, regs); kcb->kprobe_status = KPROBE_HIT_SS; return 1; @@ -358,8 +355,6 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) regs->pc = orig_ret_address; kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); - hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); kfree(ri); @@ -508,14 +503,8 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, if (post_kprobe_handler(args->regs)) ret = NOTIFY_STOP; } else { - if (kprobe_handler(args->regs)) { + if (kprobe_handler(args->regs)) ret = NOTIFY_STOP; - } else { - p = __this_cpu_read(current_kprobe); - if (p->break_handler && - p->break_handler(p, args->regs)) - ret = NOTIFY_STOP; - } } } } @@ -523,57 +512,6 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, return ret; } -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - unsigned long addr; - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - kcb->jprobe_saved_regs = *regs; - kcb->jprobe_saved_r15 = regs->regs[15]; - addr = kcb->jprobe_saved_r15; - - /* - * TBD: As Linus pointed out, gcc assumes that the callee - * owns the argument space and could overwrite it, e.g. - * tailcall optimization. So, to be absolutely safe - * we also save and restore enough stack bytes to cover - * the argument area. - */ - memcpy(kcb->jprobes_stack, (kprobe_opcode_t *) addr, - MIN_STACK_SIZE(addr)); - - regs->pc = (unsigned long)(jp->entry); - - return 1; -} - -void __kprobes jprobe_return(void) -{ - asm volatile ("trapa #0x3a\n\t" "jprobe_return_end:\n\t" "nop\n\t"); -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - unsigned long stack_addr = kcb->jprobe_saved_r15; - u8 *addr = (u8 *)regs->pc; - - if ((addr >= (u8 *)jprobe_return) && - (addr <= (u8 *)jprobe_return_end)) { - *regs = kcb->jprobe_saved_regs; - - memcpy((kprobe_opcode_t *)stack_addr, kcb->jprobes_stack, - MIN_STACK_SIZE(stack_addr)); - - kcb->kprobe_status = KPROBE_HIT_SS; - preempt_enable_no_resched(); - return 1; - } - - return 0; -} - static struct kprobe trampoline_p = { .addr = (kprobe_opcode_t *)&kretprobe_trampoline, .pre_handler = trampoline_probe_handler diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index ac67828da201..410b263ef5c8 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild @@ -13,6 +13,7 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += module.h +generic-y += msi.h generic-y += preempt.h generic-y += rwsem.h generic-y += serial.h diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index d13ce517f4b9..94c930f0bc62 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -27,17 +27,17 @@ int atomic_fetch_or(int, atomic_t *); int atomic_fetch_xor(int, atomic_t *); int atomic_cmpxchg(atomic_t *, int, int); int atomic_xchg(atomic_t *, int); -int __atomic_add_unless(atomic_t *, int, int); +int atomic_fetch_add_unless(atomic_t *, int, int); void atomic_set(atomic_t *, int); +#define atomic_fetch_add_unless atomic_fetch_add_unless + #define atomic_set_release(v, i) atomic_set((v), (i)) #define atomic_read(v) READ_ONCE((v)->counter) #define atomic_add(i, v) ((void)atomic_add_return( (int)(i), (v))) #define atomic_sub(i, v) ((void)atomic_add_return(-(int)(i), (v))) -#define atomic_inc(v) ((void)atomic_add_return( 1, (v))) -#define atomic_dec(v) ((void)atomic_add_return( -1, (v))) #define atomic_and(i, v) ((void)atomic_fetch_and((i), (v))) #define atomic_or(i, v) ((void)atomic_fetch_or((i), (v))) @@ -46,22 +46,4 @@ void atomic_set(atomic_t *, int); #define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v))) #define atomic_fetch_sub(i, v) (atomic_fetch_add (-(int)(i), (v))) -#define atomic_inc_return(v) (atomic_add_return( 1, (v))) -#define atomic_dec_return(v) (atomic_add_return( -1, (v))) - -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - #endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index 28db058d471b..6963482c81d8 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -50,38 +50,6 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define atomic_dec_return(v) atomic_sub_return(1, v) -#define atomic64_dec_return(v) atomic64_sub_return(1, v) - -#define atomic_inc_return(v) atomic_add_return(1, v) -#define atomic64_inc_return(v) atomic64_add_return(1, v) - -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) - -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) -#define atomic64_sub_and_test(i, v) (atomic64_sub_return(i, v) == 0) - -#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) -#define atomic64_dec_and_test(v) (atomic64_sub_return(1, v) == 0) - -#define atomic_inc(v) atomic_add(1, v) -#define atomic64_inc(v) atomic64_add(1, v) - -#define atomic_dec(v) atomic_sub(1, v) -#define atomic64_dec(v) atomic64_sub(1, v) - -#define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0) -#define atomic64_add_negative(i, v) (atomic64_add_return(i, v) < 0) - #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) static inline int atomic_xchg(atomic_t *v, int new) @@ -89,42 +57,11 @@ static inline int atomic_xchg(atomic_t *v, int new) return xchg(&v->counter, new); } -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) -static inline long atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - c = atomic64_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic64_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c != (u); -} - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - long atomic64_dec_if_positive(atomic64_t *v); +#define atomic64_dec_if_positive atomic64_dec_if_positive #endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h index 3704490b4488..bfcaa6326c20 100644 --- a/arch/sparc/include/asm/kprobes.h +++ b/arch/sparc/include/asm/kprobes.h @@ -44,7 +44,6 @@ struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_orig_tnpc; unsigned long kprobe_orig_tstate_pil; - struct pt_regs jprobe_saved_regs; struct prev_kprobe prev_kprobe; }; diff --git a/arch/sparc/include/asm/msi.h b/arch/sparc/include/asm/msi.h deleted file mode 100644 index 3c17c1074431..000000000000 --- a/arch/sparc/include/asm/msi.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * msi.h: Defines specific to the MBus - Sbus - Interface. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - */ - -#ifndef _SPARC_MSI_H -#define _SPARC_MSI_H - -/* - * Locations of MSI Registers. - */ -#define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */ - -/* - * Useful bits in the MSI Registers. - */ -#define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */ - - -static inline void msi_set_sync(void) -{ - __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t" - "andn %%g3, %2, %%g3\n\t" - "sta %%g3, [%0] %1\n\t" : : - "r" (MSI_MBUS_ARBEN), - "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3"); -} - -#endif /* !(_SPARC_MSI_H) */ diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index ab4ba4347941..dfbca2470536 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c @@ -147,18 +147,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) kcb->kprobe_status = KPROBE_REENTER; prepare_singlestep(p, regs, kcb); return 1; - } else { - if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { + } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* The breakpoint instruction was removed by * another cpu right after we hit, no further * handling of this interrupt is appropriate */ - ret = 1; - goto no_kprobe; - } - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) - goto ss_probe; + ret = 1; } goto no_kprobe; } @@ -181,10 +175,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) set_current_kprobe(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (p->pre_handler && p->pre_handler(p, regs)) + if (p->pre_handler && p->pre_handler(p, regs)) { + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; + } -ss_probe: prepare_singlestep(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_SS; return 1; @@ -441,53 +437,6 @@ out: exception_exit(prev_state); } -/* Jprobes support. */ -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - memcpy(&(kcb->jprobe_saved_regs), regs, sizeof(*regs)); - - regs->tpc = (unsigned long) jp->entry; - regs->tnpc = ((unsigned long) jp->entry) + 0x4UL; - regs->tstate |= TSTATE_PIL; - - return 1; -} - -void __kprobes jprobe_return(void) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - register unsigned long orig_fp asm("g1"); - - orig_fp = kcb->jprobe_saved_regs.u_regs[UREG_FP]; - __asm__ __volatile__("\n" -"1: cmp %%sp, %0\n\t" - "blu,a,pt %%xcc, 1b\n\t" - " restore\n\t" - ".globl jprobe_return_trap_instruction\n" -"jprobe_return_trap_instruction:\n\t" - "ta 0x70" - : /* no outputs */ - : "r" (orig_fp)); -} - -extern void jprobe_return_trap_instruction(void); - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - u32 *addr = (u32 *) regs->tpc; - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - if (addr == (u32 *) jprobe_return_trap_instruction) { - memcpy(regs, &(kcb->jprobe_saved_regs), sizeof(*regs)); - preempt_enable_no_resched(); - return 1; - } - return 0; -} - /* The value stored in the return address register is actually 2 * instructions before where the callee will return to. * Sequences usually look something like this @@ -562,9 +511,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, regs->tpc = orig_ret_address; regs->tnpc = orig_ret_address + 4; - reset_current_kprobe(); kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 2ef8cfa9677e..f0eba72aa1ad 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -814,7 +814,7 @@ static void __init get_tick_patch(void) } } -static void init_tick_ops(struct sparc64_tick_ops *ops) +static void __init init_tick_ops(struct sparc64_tick_ops *ops) { unsigned long freq, quotient, tick; diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 465a901a0ada..281fa634bb1a 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -95,7 +95,7 @@ int atomic_cmpxchg(atomic_t *v, int old, int new) } EXPORT_SYMBOL(atomic_cmpxchg); -int __atomic_add_unless(atomic_t *v, int a, int u) +int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int ret; unsigned long flags; @@ -107,7 +107,7 @@ int __atomic_add_unless(atomic_t *v, int a, int u) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(__atomic_add_unless); +EXPORT_SYMBOL(atomic_fetch_add_unless); /* Atomic operations are already serializing */ void atomic_set(atomic_t *v, int i) diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 1d70c3f6d986..be9cb0065179 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -37,7 +37,6 @@ #include <asm/mbus.h> #include <asm/page.h> #include <asm/asi.h> -#include <asm/msi.h> #include <asm/smp.h> #include <asm/io.h> @@ -116,6 +115,25 @@ static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) set_pte((pte_t *)ctxp, pte); } +/* + * Locations of MSI Registers. + */ +#define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */ + +/* + * Useful bits in the MSI Registers. + */ +#define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */ + +static void msi_set_sync(void) +{ + __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t" + "andn %%g3, %2, %%g3\n\t" + "sta %%g3, [%0] %1\n\t" : : + "r" (MSI_MBUS_ARBEN), + "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3"); +} + void pmd_set(pmd_t *pmdp, pte_t *ptep) { unsigned long ptp; /* Physical address, shifted right by 4 */ diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f1dbb4ee19d7..0b767d6577b7 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -63,7 +63,7 @@ config X86 select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_REFCOUNT select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 - select ARCH_HAS_UACCESS_MCSAFE if X86_64 + select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE select ARCH_HAS_SET_MEMORY select ARCH_HAS_SG_CHAIN select ARCH_HAS_STRICT_KERNEL_RWX @@ -180,13 +180,14 @@ config X86 select HAVE_PERF_USER_STACK_DUMP select HAVE_RCU_TABLE_FREE select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_RELIABLE_STACKTRACE if X86_64 && UNWINDER_FRAME_POINTER && STACK_VALIDATION + select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS select HAVE_UNSTABLE_SCHED_CLOCK select HAVE_USER_RETURN_NOTIFIER + select HOTPLUG_SMT if SMP select IRQ_FORCED_THREADING select NEED_SG_DMA_LENGTH select PCI_LOCKLESS_CONFIG diff --git a/arch/x86/Makefile b/arch/x86/Makefile index a08e82856563..7e3c07d6ad42 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -80,11 +80,6 @@ ifeq ($(CONFIG_X86_32),y) # alignment instructions. KBUILD_CFLAGS += $(call cc-option,$(cc_stack_align4)) - # Disable unit-at-a-time mode on pre-gcc-4.0 compilers, it makes gcc use - # a lot more stack due to the lack of sharing of stacklots: - KBUILD_CFLAGS += $(call cc-ifversion, -lt, 0400, \ - $(call cc-option,-fno-unit-at-a-time)) - # CPU-specific tuning. Anything which can be shared with UML should go here. include arch/x86/Makefile_32.cpu KBUILD_CFLAGS += $(cflags-y) diff --git a/arch/x86/boot/bitops.h b/arch/x86/boot/bitops.h index 0d41d68131cc..2e1382486e91 100644 --- a/arch/x86/boot/bitops.h +++ b/arch/x86/boot/bitops.h @@ -17,6 +17,7 @@ #define _LINUX_BITOPS_H /* Inhibit inclusion of <linux/bitops.h> */ #include <linux/types.h> +#include <asm/asm.h> static inline bool constant_test_bit(int nr, const void *addr) { @@ -28,7 +29,7 @@ static inline bool variable_test_bit(int nr, const void *addr) bool v; const u32 *p = (const u32 *)addr; - asm("btl %2,%1; setc %0" : "=qm" (v) : "m" (*p), "Ir" (nr)); + asm("btl %2,%1" CC_SET(c) : CC_OUT(c) (v) : "m" (*p), "Ir" (nr)); return v; } diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index fa42f895fdde..169c2feda14a 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -106,9 +106,13 @@ define cmd_check_data_rel done endef +# We need to run two commands under "if_changed", so merge them into a +# single invocation. +quiet_cmd_check-and-link-vmlinux = LD $@ + cmd_check-and-link-vmlinux = $(cmd_check_data_rel); $(cmd_ld) + $(obj)/vmlinux: $(vmlinux-objs-y) FORCE - $(call if_changed,check_data_rel) - $(call if_changed,ld) + $(call if_changed,check-and-link-vmlinux) OBJCOPYFLAGS_vmlinux.bin := -R .comment -S $(obj)/vmlinux.bin: vmlinux FORCE diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c index e57665b4ba1c..1458b1700fc7 100644 --- a/arch/x86/boot/compressed/eboot.c +++ b/arch/x86/boot/compressed/eboot.c @@ -34,74 +34,13 @@ static void setup_boot_services##bits(struct efi_config *c) \ \ table = (typeof(table))sys_table; \ \ - c->runtime_services = table->runtime; \ - c->boot_services = table->boottime; \ - c->text_output = table->con_out; \ + c->runtime_services = table->runtime; \ + c->boot_services = table->boottime; \ + c->text_output = table->con_out; \ } BOOT_SERVICES(32); BOOT_SERVICES(64); -static inline efi_status_t __open_volume32(void *__image, void **__fh) -{ - efi_file_io_interface_t *io; - efi_loaded_image_32_t *image = __image; - efi_file_handle_32_t *fh; - efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID; - efi_status_t status; - void *handle = (void *)(unsigned long)image->device_handle; - unsigned long func; - - status = efi_call_early(handle_protocol, handle, - &fs_proto, (void **)&io); - if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to handle fs_proto\n"); - return status; - } - - func = (unsigned long)io->open_volume; - status = efi_early->call(func, io, &fh); - if (status != EFI_SUCCESS) - efi_printk(sys_table, "Failed to open volume\n"); - - *__fh = fh; - return status; -} - -static inline efi_status_t __open_volume64(void *__image, void **__fh) -{ - efi_file_io_interface_t *io; - efi_loaded_image_64_t *image = __image; - efi_file_handle_64_t *fh; - efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID; - efi_status_t status; - void *handle = (void *)(unsigned long)image->device_handle; - unsigned long func; - - status = efi_call_early(handle_protocol, handle, - &fs_proto, (void **)&io); - if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to handle fs_proto\n"); - return status; - } - - func = (unsigned long)io->open_volume; - status = efi_early->call(func, io, &fh); - if (status != EFI_SUCCESS) - efi_printk(sys_table, "Failed to open volume\n"); - - *__fh = fh; - return status; -} - -efi_status_t -efi_open_volume(efi_system_table_t *sys_table, void *__image, void **__fh) -{ - if (efi_early->is64) - return __open_volume64(__image, __fh); - - return __open_volume32(__image, __fh); -} - void efi_char16_printk(efi_system_table_t *table, efi_char16_t *str) { efi_call_proto(efi_simple_text_output_protocol, output_string, @@ -109,23 +48,17 @@ void efi_char16_printk(efi_system_table_t *table, efi_char16_t *str) } static efi_status_t -__setup_efi_pci(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) +preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) { struct pci_setup_rom *rom = NULL; efi_status_t status; unsigned long size; - uint64_t attributes, romsize; + uint64_t romsize; void *romimage; - status = efi_call_proto(efi_pci_io_protocol, attributes, pci, - EfiPciIoAttributeOperationGet, 0ULL, - &attributes); - if (status != EFI_SUCCESS) - return status; - /* - * Some firmware images contain EFI function pointers at the place where the - * romimage and romsize fields are supposed to be. Typically the EFI + * Some firmware images contain EFI function pointers at the place where + * the romimage and romsize fields are supposed to be. Typically the EFI * code is mapped at high addresses, translating to an unrealistically * large romsize. The UEFI spec limits the size of option ROMs to 16 * MiB so we reject any ROMs over 16 MiB in size to catch this. @@ -140,16 +73,16 @@ __setup_efi_pci(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) status = efi_call_early(allocate_pool, EFI_LOADER_DATA, size, &rom); if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to alloc mem for rom\n"); + efi_printk(sys_table, "Failed to allocate memory for 'rom'\n"); return status; } memset(rom, 0, sizeof(*rom)); - rom->data.type = SETUP_PCI; - rom->data.len = size - sizeof(struct setup_data); - rom->data.next = 0; - rom->pcilen = pci->romsize; + rom->data.type = SETUP_PCI; + rom->data.len = size - sizeof(struct setup_data); + rom->data.next = 0; + rom->pcilen = pci->romsize; *__rom = rom; status = efi_call_proto(efi_pci_io_protocol, pci.read, pci, @@ -185,96 +118,6 @@ free_struct: return status; } -static void -setup_efi_pci32(struct boot_params *params, void **pci_handle, - unsigned long size) -{ - efi_pci_io_protocol_t *pci = NULL; - efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; - u32 *handles = (u32 *)(unsigned long)pci_handle; - efi_status_t status; - unsigned long nr_pci; - struct setup_data *data; - int i; - - data = (struct setup_data *)(unsigned long)params->hdr.setup_data; - - while (data && data->next) - data = (struct setup_data *)(unsigned long)data->next; - - nr_pci = size / sizeof(u32); - for (i = 0; i < nr_pci; i++) { - struct pci_setup_rom *rom = NULL; - u32 h = handles[i]; - - status = efi_call_early(handle_protocol, h, - &pci_proto, (void **)&pci); - - if (status != EFI_SUCCESS) - continue; - - if (!pci) - continue; - - status = __setup_efi_pci(pci, &rom); - if (status != EFI_SUCCESS) - continue; - - if (data) - data->next = (unsigned long)rom; - else - params->hdr.setup_data = (unsigned long)rom; - - data = (struct setup_data *)rom; - - } -} - -static void -setup_efi_pci64(struct boot_params *params, void **pci_handle, - unsigned long size) -{ - efi_pci_io_protocol_t *pci = NULL; - efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; - u64 *handles = (u64 *)(unsigned long)pci_handle; - efi_status_t status; - unsigned long nr_pci; - struct setup_data *data; - int i; - - data = (struct setup_data *)(unsigned long)params->hdr.setup_data; - - while (data && data->next) - data = (struct setup_data *)(unsigned long)data->next; - - nr_pci = size / sizeof(u64); - for (i = 0; i < nr_pci; i++) { - struct pci_setup_rom *rom = NULL; - u64 h = handles[i]; - - status = efi_call_early(handle_protocol, h, - &pci_proto, (void **)&pci); - - if (status != EFI_SUCCESS) - continue; - - if (!pci) - continue; - - status = __setup_efi_pci(pci, &rom); - if (status != EFI_SUCCESS) - continue; - - if (data) - data->next = (unsigned long)rom; - else - params->hdr.setup_data = (unsigned long)rom; - - data = (struct setup_data *)rom; - - } -} - /* * There's no way to return an informative status from this function, * because any analysis (and printing of error messages) needs to be @@ -290,6 +133,9 @@ static void setup_efi_pci(struct boot_params *params) void **pci_handle = NULL; efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; unsigned long size = 0; + unsigned long nr_pci; + struct setup_data *data; + int i; status = efi_call_early(locate_handle, EFI_LOCATE_BY_PROTOCOL, @@ -301,7 +147,7 @@ static void setup_efi_pci(struct boot_params *params) size, (void **)&pci_handle); if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to alloc mem for pci_handle\n"); + efi_printk(sys_table, "Failed to allocate memory for 'pci_handle'\n"); return; } @@ -313,10 +159,34 @@ static void setup_efi_pci(struct boot_params *params) if (status != EFI_SUCCESS) goto free_handle; - if (efi_early->is64) - setup_efi_pci64(params, pci_handle, size); - else - setup_efi_pci32(params, pci_handle, size); + data = (struct setup_data *)(unsigned long)params->hdr.setup_data; + + while (data && data->next) + data = (struct setup_data *)(unsigned long)data->next; + + nr_pci = size / (efi_is_64bit() ? sizeof(u64) : sizeof(u32)); + for (i = 0; i < nr_pci; i++) { + efi_pci_io_protocol_t *pci = NULL; + struct pci_setup_rom *rom; + + status = efi_call_early(handle_protocol, + efi_is_64bit() ? ((u64 *)pci_handle)[i] + : ((u32 *)pci_handle)[i], + &pci_proto, (void **)&pci); + if (status != EFI_SUCCESS || !pci) + continue; + + status = preserve_pci_rom_image(pci, &rom); + if (status != EFI_SUCCESS) + continue; + + if (data) + data->next = (unsigned long)rom; + else + params->hdr.setup_data = (unsigned long)rom; + + data = (struct setup_data *)rom; + } free_handle: efi_call_early(free_pool, pci_handle); @@ -347,8 +217,7 @@ static void retrieve_apple_device_properties(struct boot_params *boot_params) status = efi_call_early(allocate_pool, EFI_LOADER_DATA, size + sizeof(struct setup_data), &new); if (status != EFI_SUCCESS) { - efi_printk(sys_table, - "Failed to alloc mem for properties\n"); + efi_printk(sys_table, "Failed to allocate memory for 'properties'\n"); return; } @@ -364,9 +233,9 @@ static void retrieve_apple_device_properties(struct boot_params *boot_params) new->next = 0; data = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data; - if (!data) + if (!data) { boot_params->hdr.setup_data = (unsigned long)new; - else { + } else { while (data->next) data = (struct setup_data *)(unsigned long)data->next; data->next = (unsigned long)new; @@ -386,81 +255,55 @@ static void setup_quirks(struct boot_params *boot_params) } } +/* + * See if we have Universal Graphics Adapter (UGA) protocol + */ static efi_status_t -setup_uga32(void **uga_handle, unsigned long size, u32 *width, u32 *height) +setup_uga(struct screen_info *si, efi_guid_t *uga_proto, unsigned long size) { - struct efi_uga_draw_protocol *uga = NULL, *first_uga; - efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID; + efi_status_t status; + u32 width, height; + void **uga_handle = NULL; + efi_uga_draw_protocol_t *uga = NULL, *first_uga; unsigned long nr_ugas; - u32 *handles = (u32 *)uga_handle; - efi_status_t status = EFI_INVALID_PARAMETER; int i; - first_uga = NULL; - nr_ugas = size / sizeof(u32); - for (i = 0; i < nr_ugas; i++) { - efi_guid_t pciio_proto = EFI_PCI_IO_PROTOCOL_GUID; - u32 w, h, depth, refresh; - void *pciio; - u32 handle = handles[i]; - - status = efi_call_early(handle_protocol, handle, - &uga_proto, (void **)&uga); - if (status != EFI_SUCCESS) - continue; - - efi_call_early(handle_protocol, handle, &pciio_proto, &pciio); - - status = efi_early->call((unsigned long)uga->get_mode, uga, - &w, &h, &depth, &refresh); - if (status == EFI_SUCCESS && (!first_uga || pciio)) { - *width = w; - *height = h; - - /* - * Once we've found a UGA supporting PCIIO, - * don't bother looking any further. - */ - if (pciio) - break; - - first_uga = uga; - } - } + status = efi_call_early(allocate_pool, EFI_LOADER_DATA, + size, (void **)&uga_handle); + if (status != EFI_SUCCESS) + return status; - return status; -} + status = efi_call_early(locate_handle, + EFI_LOCATE_BY_PROTOCOL, + uga_proto, NULL, &size, uga_handle); + if (status != EFI_SUCCESS) + goto free_handle; -static efi_status_t -setup_uga64(void **uga_handle, unsigned long size, u32 *width, u32 *height) -{ - struct efi_uga_draw_protocol *uga = NULL, *first_uga; - efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID; - unsigned long nr_ugas; - u64 *handles = (u64 *)uga_handle; - efi_status_t status = EFI_INVALID_PARAMETER; - int i; + height = 0; + width = 0; first_uga = NULL; - nr_ugas = size / sizeof(u64); + nr_ugas = size / (efi_is_64bit() ? sizeof(u64) : sizeof(u32)); for (i = 0; i < nr_ugas; i++) { efi_guid_t pciio_proto = EFI_PCI_IO_PROTOCOL_GUID; u32 w, h, depth, refresh; void *pciio; - u64 handle = handles[i]; + unsigned long handle = efi_is_64bit() ? ((u64 *)uga_handle)[i] + : ((u32 *)uga_handle)[i]; status = efi_call_early(handle_protocol, handle, - &uga_proto, (void **)&uga); + uga_proto, (void **)&uga); if (status != EFI_SUCCESS) continue; + pciio = NULL; efi_call_early(handle_protocol, handle, &pciio_proto, &pciio); - status = efi_early->call((unsigned long)uga->get_mode, uga, - &w, &h, &depth, &refresh); + status = efi_call_proto(efi_uga_draw_protocol, get_mode, uga, + &w, &h, &depth, &refresh); if (status == EFI_SUCCESS && (!first_uga || pciio)) { - *width = w; - *height = h; + width = w; + height = h; /* * Once we've found a UGA supporting PCIIO, @@ -473,59 +316,28 @@ setup_uga64(void **uga_handle, unsigned long size, u32 *width, u32 *height) } } - return status; -} - -/* - * See if we have Universal Graphics Adapter (UGA) protocol - */ -static efi_status_t setup_uga(struct screen_info *si, efi_guid_t *uga_proto, - unsigned long size) -{ - efi_status_t status; - u32 width, height; - void **uga_handle = NULL; - - status = efi_call_early(allocate_pool, EFI_LOADER_DATA, - size, (void **)&uga_handle); - if (status != EFI_SUCCESS) - return status; - - status = efi_call_early(locate_handle, - EFI_LOCATE_BY_PROTOCOL, - uga_proto, NULL, &size, uga_handle); - if (status != EFI_SUCCESS) - goto free_handle; - - height = 0; - width = 0; - - if (efi_early->is64) - status = setup_uga64(uga_handle, size, &width, &height); - else - status = setup_uga32(uga_handle, size, &width, &height); - if (!width && !height) goto free_handle; /* EFI framebuffer */ - si->orig_video_isVGA = VIDEO_TYPE_EFI; + si->orig_video_isVGA = VIDEO_TYPE_EFI; - si->lfb_depth = 32; - si->lfb_width = width; - si->lfb_height = height; + si->lfb_depth = 32; + si->lfb_width = width; + si->lfb_height = height; - si->red_size = 8; - si->red_pos = 16; - si->green_size = 8; - si->green_pos = 8; - si->blue_size = 8; - si->blue_pos = 0; - si->rsvd_size = 8; - si->rsvd_pos = 24; + si->red_size = 8; + si->red_pos = 16; + si->green_size = 8; + si->green_pos = 8; + si->blue_size = 8; + si->blue_pos = 0; + si->rsvd_size = 8; + si->rsvd_pos = 24; free_handle: efi_call_early(free_pool, uga_handle); + return status; } @@ -592,7 +404,7 @@ struct boot_params *make_boot_params(struct efi_config *c) if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) return NULL; - if (efi_early->is64) + if (efi_is_64bit()) setup_boot_services64(efi_early); else setup_boot_services32(efi_early); @@ -607,7 +419,7 @@ struct boot_params *make_boot_params(struct efi_config *c) status = efi_low_alloc(sys_table, 0x4000, 1, (unsigned long *)&boot_params); if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to alloc lowmem for boot params\n"); + efi_printk(sys_table, "Failed to allocate lowmem for boot params\n"); return NULL; } @@ -623,9 +435,9 @@ struct boot_params *make_boot_params(struct efi_config *c) * Fill out some of the header fields ourselves because the * EFI firmware loader doesn't load the first sector. */ - hdr->root_flags = 1; - hdr->vid_mode = 0xffff; - hdr->boot_flag = 0xAA55; + hdr->root_flags = 1; + hdr->vid_mode = 0xffff; + hdr->boot_flag = 0xAA55; hdr->type_of_loader = 0x21; @@ -633,6 +445,7 @@ struct boot_params *make_boot_params(struct efi_config *c) cmdline_ptr = efi_convert_cmdline(sys_table, image, &options_size); if (!cmdline_ptr) goto fail; + hdr->cmd_line_ptr = (unsigned long)cmdline_ptr; /* Fill in upper bits of command line address, NOP on 32 bit */ boot_params->ext_cmd_line_ptr = (u64)(unsigned long)cmdline_ptr >> 32; @@ -669,10 +482,12 @@ struct boot_params *make_boot_params(struct efi_config *c) boot_params->ext_ramdisk_size = (u64)ramdisk_size >> 32; return boot_params; + fail2: efi_free(sys_table, options_size, hdr->cmd_line_ptr); fail: efi_free(sys_table, 0x4000, (unsigned long)boot_params); + return NULL; } @@ -684,7 +499,7 @@ static void add_e820ext(struct boot_params *params, unsigned long size; e820ext->type = SETUP_E820_EXT; - e820ext->len = nr_entries * sizeof(struct boot_e820_entry); + e820ext->len = nr_entries * sizeof(struct boot_e820_entry); e820ext->next = 0; data = (struct setup_data *)(unsigned long)params->hdr.setup_data; @@ -698,8 +513,8 @@ static void add_e820ext(struct boot_params *params, params->hdr.setup_data = (unsigned long)e820ext; } -static efi_status_t setup_e820(struct boot_params *params, - struct setup_data *e820ext, u32 e820ext_size) +static efi_status_t +setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_size) { struct boot_e820_entry *entry = params->e820_table; struct efi_info *efi = ¶ms->efi_info; @@ -820,11 +635,10 @@ static efi_status_t alloc_e820ext(u32 nr_desc, struct setup_data **e820ext, } struct exit_boot_struct { - struct boot_params *boot_params; - struct efi_info *efi; - struct setup_data *e820ext; - __u32 e820ext_size; - bool is64; + struct boot_params *boot_params; + struct efi_info *efi; + struct setup_data *e820ext; + __u32 e820ext_size; }; static efi_status_t exit_boot_func(efi_system_table_t *sys_table_arg, @@ -851,25 +665,25 @@ static efi_status_t exit_boot_func(efi_system_table_t *sys_table_arg, first = false; } - signature = p->is64 ? EFI64_LOADER_SIGNATURE : EFI32_LOADER_SIGNATURE; + signature = efi_is_64bit() ? EFI64_LOADER_SIGNATURE + : EFI32_LOADER_SIGNATURE; memcpy(&p->efi->efi_loader_signature, signature, sizeof(__u32)); - p->efi->efi_systab = (unsigned long)sys_table_arg; - p->efi->efi_memdesc_size = *map->desc_size; - p->efi->efi_memdesc_version = *map->desc_ver; - p->efi->efi_memmap = (unsigned long)*map->map; - p->efi->efi_memmap_size = *map->map_size; + p->efi->efi_systab = (unsigned long)sys_table_arg; + p->efi->efi_memdesc_size = *map->desc_size; + p->efi->efi_memdesc_version = *map->desc_ver; + p->efi->efi_memmap = (unsigned long)*map->map; + p->efi->efi_memmap_size = *map->map_size; #ifdef CONFIG_X86_64 - p->efi->efi_systab_hi = (unsigned long)sys_table_arg >> 32; - p->efi->efi_memmap_hi = (unsigned long)*map->map >> 32; + p->efi->efi_systab_hi = (unsigned long)sys_table_arg >> 32; + p->efi->efi_memmap_hi = (unsigned long)*map->map >> 32; #endif return EFI_SUCCESS; } -static efi_status_t exit_boot(struct boot_params *boot_params, - void *handle, bool is64) +static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) { unsigned long map_sz, key, desc_size, buff_size; efi_memory_desc_t *mem_map; @@ -880,17 +694,16 @@ static efi_status_t exit_boot(struct boot_params *boot_params, struct efi_boot_memmap map; struct exit_boot_struct priv; - map.map = &mem_map; - map.map_size = &map_sz; - map.desc_size = &desc_size; - map.desc_ver = &desc_version; - map.key_ptr = &key; - map.buff_size = &buff_size; - priv.boot_params = boot_params; - priv.efi = &boot_params->efi_info; - priv.e820ext = NULL; - priv.e820ext_size = 0; - priv.is64 = is64; + map.map = &mem_map; + map.map_size = &map_sz; + map.desc_size = &desc_size; + map.desc_ver = &desc_version; + map.key_ptr = &key; + map.buff_size = &buff_size; + priv.boot_params = boot_params; + priv.efi = &boot_params->efi_info; + priv.e820ext = NULL; + priv.e820ext_size = 0; /* Might as well exit boot services now */ status = efi_exit_boot_services(sys_table, handle, &map, &priv, @@ -898,10 +711,11 @@ static efi_status_t exit_boot(struct boot_params *boot_params, if (status != EFI_SUCCESS) return status; - e820ext = priv.e820ext; - e820ext_size = priv.e820ext_size; + e820ext = priv.e820ext; + e820ext_size = priv.e820ext_size; + /* Historic? */ - boot_params->alt_mem_k = 32 * 1024; + boot_params->alt_mem_k = 32 * 1024; status = setup_e820(boot_params, e820ext, e820ext_size); if (status != EFI_SUCCESS) @@ -914,8 +728,8 @@ static efi_status_t exit_boot(struct boot_params *boot_params, * On success we return a pointer to a boot_params structure, and NULL * on failure. */ -struct boot_params *efi_main(struct efi_config *c, - struct boot_params *boot_params) +struct boot_params * +efi_main(struct efi_config *c, struct boot_params *boot_params) { struct desc_ptr *gdt = NULL; efi_loaded_image_t *image; @@ -924,13 +738,11 @@ struct boot_params *efi_main(struct efi_config *c, struct desc_struct *desc; void *handle; efi_system_table_t *_table; - bool is64; efi_early = c; _table = (efi_system_table_t *)(unsigned long)efi_early->table; handle = (void *)(unsigned long)efi_early->image_handle; - is64 = efi_early->is64; sys_table = _table; @@ -938,7 +750,7 @@ struct boot_params *efi_main(struct efi_config *c, if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) goto fail; - if (is64) + if (efi_is_64bit()) setup_boot_services64(efi_early); else setup_boot_services32(efi_early); @@ -963,7 +775,7 @@ struct boot_params *efi_main(struct efi_config *c, status = efi_call_early(allocate_pool, EFI_LOADER_DATA, sizeof(*gdt), (void **)&gdt); if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to alloc mem for gdt structure\n"); + efi_printk(sys_table, "Failed to allocate memory for 'gdt' structure\n"); goto fail; } @@ -971,7 +783,7 @@ struct boot_params *efi_main(struct efi_config *c, status = efi_low_alloc(sys_table, gdt->size, 8, (unsigned long *)&gdt->address); if (status != EFI_SUCCESS) { - efi_printk(sys_table, "Failed to alloc mem for gdt\n"); + efi_printk(sys_table, "Failed to allocate memory for 'gdt'\n"); goto fail; } @@ -994,7 +806,7 @@ struct boot_params *efi_main(struct efi_config *c, hdr->code32_start = bzimage_addr; } - status = exit_boot(boot_params, handle, is64); + status = exit_boot(boot_params, handle); if (status != EFI_SUCCESS) { efi_printk(sys_table, "exit_boot() failed!\n"); goto fail; @@ -1008,19 +820,20 @@ struct boot_params *efi_main(struct efi_config *c, if (IS_ENABLED(CONFIG_X86_64)) { /* __KERNEL32_CS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; - desc->l = 0; - desc->d = SEG_OP_SIZE_32BIT; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; + desc->limit0 = 0xffff; + desc->base0 = 0x0000; + desc->base1 = 0x0000; + desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; + desc->s = DESC_TYPE_CODE_DATA; + desc->dpl = 0; + desc->p = 1; + desc->limit1 = 0xf; + desc->avl = 0; + desc->l = 0; + desc->d = SEG_OP_SIZE_32BIT; + desc->g = SEG_GRANULARITY_4KB; + desc->base2 = 0x00; + desc++; } else { /* Second entry is unused on 32-bit */ @@ -1028,15 +841,16 @@ struct boot_params *efi_main(struct efi_config *c, } /* __KERNEL_CS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; + desc->limit0 = 0xffff; + desc->base0 = 0x0000; + desc->base1 = 0x0000; + desc->type = SEG_TYPE_CODE | SEG_TYPE_EXEC_READ; + desc->s = DESC_TYPE_CODE_DATA; + desc->dpl = 0; + desc->p = 1; + desc->limit1 = 0xf; + desc->avl = 0; + if (IS_ENABLED(CONFIG_X86_64)) { desc->l = 1; desc->d = 0; @@ -1044,41 +858,41 @@ struct boot_params *efi_main(struct efi_config *c, desc->l = 0; desc->d = SEG_OP_SIZE_32BIT; } - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; + desc->g = SEG_GRANULARITY_4KB; + desc->base2 = 0x00; desc++; /* __KERNEL_DS */ - desc->limit0 = 0xffff; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_DATA | SEG_TYPE_READ_WRITE; - desc->s = DESC_TYPE_CODE_DATA; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0xf; - desc->avl = 0; - desc->l = 0; - desc->d = SEG_OP_SIZE_32BIT; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; + desc->limit0 = 0xffff; + desc->base0 = 0x0000; + desc->base1 = 0x0000; + desc->type = SEG_TYPE_DATA | SEG_TYPE_READ_WRITE; + desc->s = DESC_TYPE_CODE_DATA; + desc->dpl = 0; + desc->p = 1; + desc->limit1 = 0xf; + desc->avl = 0; + desc->l = 0; + desc->d = SEG_OP_SIZE_32BIT; + desc->g = SEG_GRANULARITY_4KB; + desc->base2 = 0x00; desc++; if (IS_ENABLED(CONFIG_X86_64)) { /* Task segment value */ - desc->limit0 = 0x0000; - desc->base0 = 0x0000; - desc->base1 = 0x0000; - desc->type = SEG_TYPE_TSS; - desc->s = 0; - desc->dpl = 0; - desc->p = 1; - desc->limit1 = 0x0; - desc->avl = 0; - desc->l = 0; - desc->d = 0; - desc->g = SEG_GRANULARITY_4KB; - desc->base2 = 0x00; + desc->limit0 = 0x0000; + desc->base0 = 0x0000; + desc->base1 = 0x0000; + desc->type = SEG_TYPE_TSS; + desc->s = 0; + desc->dpl = 0; + desc->p = 1; + desc->limit1 = 0x0; + desc->avl = 0; + desc->l = 0; + desc->d = 0; + desc->g = SEG_GRANULARITY_4KB; + desc->base2 = 0x00; desc++; } @@ -1088,5 +902,6 @@ struct boot_params *efi_main(struct efi_config *c, return boot_params; fail: efi_printk(sys_table, "efi_main() failed!\n"); + return NULL; } diff --git a/arch/x86/boot/compressed/eboot.h b/arch/x86/boot/compressed/eboot.h index e799dc5c6448..8297387c4676 100644 --- a/arch/x86/boot/compressed/eboot.h +++ b/arch/x86/boot/compressed/eboot.h @@ -12,22 +12,22 @@ #define DESC_TYPE_CODE_DATA (1 << 0) -struct efi_uga_draw_protocol_32 { +typedef struct { u32 get_mode; u32 set_mode; u32 blt; -}; +} efi_uga_draw_protocol_32_t; -struct efi_uga_draw_protocol_64 { +typedef struct { u64 get_mode; u64 set_mode; u64 blt; -}; +} efi_uga_draw_protocol_64_t; -struct efi_uga_draw_protocol { +typedef struct { void *get_mode; void *set_mode; void *blt; -}; +} efi_uga_draw_protocol_t; #endif /* BOOT_COMPRESSED_EBOOT_H */ diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index b87a7582853d..302517929932 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -102,7 +102,7 @@ static bool memmap_too_large; /* Store memory limit specified by "mem=nn[KMG]" or "memmap=nn[KMG]" */ -unsigned long long mem_limit = ULLONG_MAX; +static unsigned long long mem_limit = ULLONG_MAX; enum mem_avoid_index { @@ -215,7 +215,36 @@ static void mem_avoid_memmap(char *str) memmap_too_large = true; } -static int handle_mem_memmap(void) +/* Store the number of 1GB huge pages which users specified: */ +static unsigned long max_gb_huge_pages; + +static void parse_gb_huge_pages(char *param, char *val) +{ + static bool gbpage_sz; + char *p; + + if (!strcmp(param, "hugepagesz")) { + p = val; + if (memparse(p, &p) != PUD_SIZE) { + gbpage_sz = false; + return; + } + + if (gbpage_sz) + warn("Repeatedly set hugeTLB page size of 1G!\n"); + gbpage_sz = true; + return; + } + + if (!strcmp(param, "hugepages") && gbpage_sz) { + p = val; + max_gb_huge_pages = simple_strtoull(p, &p, 0); + return; + } +} + + +static int handle_mem_options(void) { char *args = (char *)get_cmd_line_ptr(); size_t len = strlen((char *)args); @@ -223,7 +252,8 @@ static int handle_mem_memmap(void) char *param, *val; u64 mem_size; - if (!strstr(args, "memmap=") && !strstr(args, "mem=")) + if (!strstr(args, "memmap=") && !strstr(args, "mem=") && + !strstr(args, "hugepages")) return 0; tmp_cmdline = malloc(len + 1); @@ -248,6 +278,8 @@ static int handle_mem_memmap(void) if (!strcmp(param, "memmap")) { mem_avoid_memmap(val); + } else if (strstr(param, "hugepages")) { + parse_gb_huge_pages(param, val); } else if (!strcmp(param, "mem")) { char *p = val; @@ -387,7 +419,7 @@ static void mem_avoid_init(unsigned long input, unsigned long input_size, /* We don't need to set a mapping for setup_data. */ /* Mark the memmap regions we need to avoid */ - handle_mem_memmap(); + handle_mem_options(); #ifdef CONFIG_X86_VERBOSE_BOOTUP /* Make sure video RAM can be used. */ @@ -466,6 +498,60 @@ static void store_slot_info(struct mem_vector *region, unsigned long image_size) } } +/* + * Skip as many 1GB huge pages as possible in the passed region + * according to the number which users specified: + */ +static void +process_gb_huge_pages(struct mem_vector *region, unsigned long image_size) +{ + unsigned long addr, size = 0; + struct mem_vector tmp; + int i = 0; + + if (!max_gb_huge_pages) { + store_slot_info(region, image_size); + return; + } + + addr = ALIGN(region->start, PUD_SIZE); + /* Did we raise the address above the passed in memory entry? */ + if (addr < region->start + region->size) + size = region->size - (addr - region->start); + + /* Check how many 1GB huge pages can be filtered out: */ + while (size > PUD_SIZE && max_gb_huge_pages) { + size -= PUD_SIZE; + max_gb_huge_pages--; + i++; + } + + /* No good 1GB huge pages found: */ + if (!i) { + store_slot_info(region, image_size); + return; + } + + /* + * Skip those 'i'*1GB good huge pages, and continue checking and + * processing the remaining head or tail part of the passed region + * if available. + */ + + if (addr >= region->start + image_size) { + tmp.start = region->start; + tmp.size = addr - region->start; + store_slot_info(&tmp, image_size); + } + + size = region->size - (addr - region->start) - i * PUD_SIZE; + if (size >= image_size) { + tmp.start = addr + i * PUD_SIZE; + tmp.size = size; + store_slot_info(&tmp, image_size); + } +} + static unsigned long slots_fetch_random(void) { unsigned long slot; @@ -546,7 +632,7 @@ static void process_mem_region(struct mem_vector *entry, /* If nothing overlaps, store the region and return. */ if (!mem_avoid_overlap(®ion, &overlap)) { - store_slot_info(®ion, image_size); + process_gb_huge_pages(®ion, image_size); return; } @@ -556,7 +642,7 @@ static void process_mem_region(struct mem_vector *entry, beginning.start = region.start; beginning.size = overlap.start - region.start; - store_slot_info(&beginning, image_size); + process_gb_huge_pages(&beginning, image_size); } /* Return if overlap extends to or past end of region. */ diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c index 8c5107545251..9e2157371491 100644 --- a/arch/x86/boot/compressed/pgtable_64.c +++ b/arch/x86/boot/compressed/pgtable_64.c @@ -1,3 +1,4 @@ +#include <asm/e820/types.h> #include <asm/processor.h> #include "pgtable.h" #include "../string.h" @@ -34,10 +35,62 @@ unsigned long *trampoline_32bit __section(.data); extern struct boot_params *boot_params; int cmdline_find_option_bool(const char *option); +static unsigned long find_trampoline_placement(void) +{ + unsigned long bios_start, ebda_start; + unsigned long trampoline_start; + struct boot_e820_entry *entry; + int i; + + /* + * Find a suitable spot for the trampoline. + * This code is based on reserve_bios_regions(). + */ + + ebda_start = *(unsigned short *)0x40e << 4; + bios_start = *(unsigned short *)0x413 << 10; + + if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) + bios_start = BIOS_START_MAX; + + if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) + bios_start = ebda_start; + + bios_start = round_down(bios_start, PAGE_SIZE); + + /* Find the first usable memory region under bios_start. */ + for (i = boot_params->e820_entries - 1; i >= 0; i--) { + entry = &boot_params->e820_table[i]; + + /* Skip all entries above bios_start. */ + if (bios_start <= entry->addr) + continue; + + /* Skip non-RAM entries. */ + if (entry->type != E820_TYPE_RAM) + continue; + + /* Adjust bios_start to the end of the entry if needed. */ + if (bios_start > entry->addr + entry->size) + bios_start = entry->addr + entry->size; + + /* Keep bios_start page-aligned. */ + bios_start = round_down(bios_start, PAGE_SIZE); + + /* Skip the entry if it's too small. */ + if (bios_start - TRAMPOLINE_32BIT_SIZE < entry->addr) + continue; + + break; + } + + /* Place the trampoline just below the end of low memory */ + return bios_start - TRAMPOLINE_32BIT_SIZE; +} + struct paging_config paging_prepare(void *rmode) { struct paging_config paging_config = {}; - unsigned long bios_start, ebda_start; /* Initialize boot_params. Required for cmdline_find_option_bool(). */ boot_params = rmode; @@ -61,23 +114,7 @@ struct paging_config paging_prepare(void *rmode) paging_config.l5_required = 1; } - /* - * Find a suitable spot for the trampoline. - * This code is based on reserve_bios_regions(). - */ - - ebda_start = *(unsigned short *)0x40e << 4; - bios_start = *(unsigned short *)0x413 << 10; - - if (bios_start < BIOS_START_MIN || bios_start > BIOS_START_MAX) - bios_start = BIOS_START_MAX; - - if (ebda_start > BIOS_START_MIN && ebda_start < bios_start) - bios_start = ebda_start; - - /* Place the trampoline just below the end of low memory, aligned to 4k */ - paging_config.trampoline_start = bios_start - TRAMPOLINE_32BIT_SIZE; - paging_config.trampoline_start = round_down(paging_config.trampoline_start, PAGE_SIZE); + paging_config.trampoline_start = find_trampoline_placement(); trampoline_32bit = (unsigned long *)paging_config.trampoline_start; diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index 16f49123d747..c4428a176973 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -13,6 +13,7 @@ */ #include <linux/types.h> +#include <asm/asm.h> #include "ctype.h" #include "string.h" @@ -28,8 +29,8 @@ int memcmp(const void *s1, const void *s2, size_t len) { bool diff; - asm("repe; cmpsb; setnz %0" - : "=qm" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + asm("repe; cmpsb" CC_SET(nz) + : CC_OUT(nz) (diff), "+D" (s1), "+S" (s2), "+c" (len)); return diff; } diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S index 9254e0b6cc06..5f7e43d4f64a 100644 --- a/arch/x86/crypto/aegis128-aesni-asm.S +++ b/arch/x86/crypto/aegis128-aesni-asm.S @@ -75,7 +75,7 @@ * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d pxor MSG, MSG mov LEN, %r8 @@ -535,6 +535,7 @@ ENTRY(crypto_aegis128_aesni_enc_tail) movdqu STATE3, 0x40(STATEP) FRAME_END + ret ENDPROC(crypto_aegis128_aesni_enc_tail) .macro decrypt_block a s0 s1 s2 s3 s4 i diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 5de7c0d46edf..acd11b3bf639 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -375,16 +375,12 @@ static struct aead_alg crypto_aegis128_aesni_alg[] = { } }; -static const struct x86_cpu_id aesni_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_AES), - X86_FEATURE_MATCH(X86_FEATURE_XMM2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); - static int __init crypto_aegis128_aesni_module_init(void) { - if (!x86_match_cpu(aesni_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_XMM2) || + !boot_cpu_has(X86_FEATURE_AES) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; return crypto_register_aeads(crypto_aegis128_aesni_alg, diff --git a/arch/x86/crypto/aegis128l-aesni-asm.S b/arch/x86/crypto/aegis128l-aesni-asm.S index 9263c344f2c7..491dd61c845c 100644 --- a/arch/x86/crypto/aegis128l-aesni-asm.S +++ b/arch/x86/crypto/aegis128l-aesni-asm.S @@ -66,7 +66,7 @@ * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d pxor MSG0, MSG0 pxor MSG1, MSG1 @@ -645,6 +645,7 @@ ENTRY(crypto_aegis128l_aesni_enc_tail) state_store0 FRAME_END + ret ENDPROC(crypto_aegis128l_aesni_enc_tail) /* diff --git a/arch/x86/crypto/aegis128l-aesni-glue.c b/arch/x86/crypto/aegis128l-aesni-glue.c index 876e4866e633..2071c3d1ae07 100644 --- a/arch/x86/crypto/aegis128l-aesni-glue.c +++ b/arch/x86/crypto/aegis128l-aesni-glue.c @@ -375,16 +375,12 @@ static struct aead_alg crypto_aegis128l_aesni_alg[] = { } }; -static const struct x86_cpu_id aesni_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_AES), - X86_FEATURE_MATCH(X86_FEATURE_XMM2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); - static int __init crypto_aegis128l_aesni_module_init(void) { - if (!x86_match_cpu(aesni_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_XMM2) || + !boot_cpu_has(X86_FEATURE_AES) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; return crypto_register_aeads(crypto_aegis128l_aesni_alg, diff --git a/arch/x86/crypto/aegis256-aesni-asm.S b/arch/x86/crypto/aegis256-aesni-asm.S index 1d977d515bf9..8870c7c5d9a4 100644 --- a/arch/x86/crypto/aegis256-aesni-asm.S +++ b/arch/x86/crypto/aegis256-aesni-asm.S @@ -59,7 +59,7 @@ * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d pxor MSG, MSG mov LEN, %r8 @@ -543,6 +543,7 @@ ENTRY(crypto_aegis256_aesni_enc_tail) state_store0 FRAME_END + ret ENDPROC(crypto_aegis256_aesni_enc_tail) /* diff --git a/arch/x86/crypto/aegis256-aesni-glue.c b/arch/x86/crypto/aegis256-aesni-glue.c index 2b5dd3af8f4d..b5f2a8fd5a71 100644 --- a/arch/x86/crypto/aegis256-aesni-glue.c +++ b/arch/x86/crypto/aegis256-aesni-glue.c @@ -375,16 +375,12 @@ static struct aead_alg crypto_aegis256_aesni_alg[] = { } }; -static const struct x86_cpu_id aesni_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_AES), - X86_FEATURE_MATCH(X86_FEATURE_XMM2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); - static int __init crypto_aegis256_aesni_module_init(void) { - if (!x86_match_cpu(aesni_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_XMM2) || + !boot_cpu_has(X86_FEATURE_AES) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; return crypto_register_aeads(crypto_aegis256_aesni_alg, diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index e762ef417562..9bd139569b41 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -258,7 +258,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff .macro GCM_INIT Iv SUBKEY AAD AADLEN mov \AADLEN, %r11 mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length - xor %r11, %r11 + xor %r11d, %r11d mov %r11, InLen(%arg2) # ctx_data.in_length = 0 mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0 mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0 @@ -286,7 +286,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff movdqu HashKey(%arg2), %xmm13 add %arg5, InLen(%arg2) - xor %r11, %r11 # initialise the data pointer offset as zero + xor %r11d, %r11d # initialise the data pointer offset as zero PARTIAL_BLOCK %arg3 %arg4 %arg5 %r11 %xmm8 \operation sub %r11, %arg5 # sub partial block data used @@ -702,7 +702,7 @@ _no_extra_mask_1_\@: # GHASH computation for the last <16 Byte block GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 - xor %rax,%rax + xor %eax, %eax mov %rax, PBlockLen(%arg2) jmp _dec_done_\@ @@ -737,7 +737,7 @@ _no_extra_mask_2_\@: # GHASH computation for the last <16 Byte block GHASH_MUL \AAD_HASH, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 - xor %rax,%rax + xor %eax, %eax mov %rax, PBlockLen(%arg2) jmp _encode_done_\@ diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index faecb1518bf8..1985ea0b551b 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -463,7 +463,7 @@ _get_AAD_rest_final\@: _get_AAD_done\@: # initialize the data pointer offset as zero - xor %r11, %r11 + xor %r11d, %r11d # start AES for num_initial_blocks blocks mov arg5, %rax # rax = *Y0 @@ -1770,7 +1770,7 @@ _get_AAD_rest_final\@: _get_AAD_done\@: # initialize the data pointer offset as zero - xor %r11, %r11 + xor %r11d, %r11d # start AES for num_initial_blocks blocks mov arg5, %rax # rax = *Y0 diff --git a/arch/x86/crypto/morus1280-avx2-asm.S b/arch/x86/crypto/morus1280-avx2-asm.S index 37d422e77931..de182c460f82 100644 --- a/arch/x86/crypto/morus1280-avx2-asm.S +++ b/arch/x86/crypto/morus1280-avx2-asm.S @@ -113,7 +113,7 @@ ENDPROC(__morus1280_update_zero) * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d vpxor MSG, MSG, MSG mov %rcx, %r8 @@ -453,6 +453,7 @@ ENTRY(crypto_morus1280_avx2_enc_tail) vmovdqu STATE4, (4 * 32)(%rdi) FRAME_END + ret ENDPROC(crypto_morus1280_avx2_enc_tail) /* diff --git a/arch/x86/crypto/morus1280-avx2-glue.c b/arch/x86/crypto/morus1280-avx2-glue.c index f111f36d26dc..6634907d6ccd 100644 --- a/arch/x86/crypto/morus1280-avx2-glue.c +++ b/arch/x86/crypto/morus1280-avx2-glue.c @@ -37,15 +37,11 @@ asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor, MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400); -static const struct x86_cpu_id avx2_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_AVX2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, avx2_cpu_id); - static int __init crypto_morus1280_avx2_module_init(void) { - if (!x86_match_cpu(avx2_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_AVX2) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) return -ENODEV; return crypto_register_aeads(crypto_morus1280_avx2_algs, diff --git a/arch/x86/crypto/morus1280-sse2-asm.S b/arch/x86/crypto/morus1280-sse2-asm.S index 1fe637c7be9d..da5d2905db60 100644 --- a/arch/x86/crypto/morus1280-sse2-asm.S +++ b/arch/x86/crypto/morus1280-sse2-asm.S @@ -235,7 +235,7 @@ ENDPROC(__morus1280_update_zero) * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d pxor MSG_LO, MSG_LO pxor MSG_HI, MSG_HI @@ -652,6 +652,7 @@ ENTRY(crypto_morus1280_sse2_enc_tail) movdqu STATE4_HI, (9 * 16)(%rdi) FRAME_END + ret ENDPROC(crypto_morus1280_sse2_enc_tail) /* diff --git a/arch/x86/crypto/morus1280-sse2-glue.c b/arch/x86/crypto/morus1280-sse2-glue.c index 839270aa713c..95cf857d2cbb 100644 --- a/arch/x86/crypto/morus1280-sse2-glue.c +++ b/arch/x86/crypto/morus1280-sse2-glue.c @@ -37,15 +37,11 @@ asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor, MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350); -static const struct x86_cpu_id sse2_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_XMM2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, sse2_cpu_id); - static int __init crypto_morus1280_sse2_module_init(void) { - if (!x86_match_cpu(sse2_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_XMM2) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; return crypto_register_aeads(crypto_morus1280_sse2_algs, diff --git a/arch/x86/crypto/morus640-sse2-asm.S b/arch/x86/crypto/morus640-sse2-asm.S index 71c72a0a0862..414db480250e 100644 --- a/arch/x86/crypto/morus640-sse2-asm.S +++ b/arch/x86/crypto/morus640-sse2-asm.S @@ -113,7 +113,7 @@ ENDPROC(__morus640_update_zero) * %r9 */ __load_partial: - xor %r9, %r9 + xor %r9d, %r9d pxor MSG, MSG mov %rcx, %r8 @@ -437,6 +437,7 @@ ENTRY(crypto_morus640_sse2_enc_tail) movdqu STATE4, (4 * 16)(%rdi) FRAME_END + ret ENDPROC(crypto_morus640_sse2_enc_tail) /* diff --git a/arch/x86/crypto/morus640-sse2-glue.c b/arch/x86/crypto/morus640-sse2-glue.c index 26b47e2db8d2..615fb7bc9a32 100644 --- a/arch/x86/crypto/morus640-sse2-glue.c +++ b/arch/x86/crypto/morus640-sse2-glue.c @@ -37,15 +37,11 @@ asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor, MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400); -static const struct x86_cpu_id sse2_cpu_id[] = { - X86_FEATURE_MATCH(X86_FEATURE_XMM2), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, sse2_cpu_id); - static int __init crypto_morus640_sse2_module_init(void) { - if (!x86_match_cpu(sse2_cpu_id)) + if (!boot_cpu_has(X86_FEATURE_XMM2) || + !boot_cpu_has(X86_FEATURE_OSXSAVE) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; return crypto_register_aeads(crypto_morus640_sse2_algs, diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S index 6204bd53528c..613d0bfc3d84 100644 --- a/arch/x86/crypto/sha1_ssse3_asm.S +++ b/arch/x86/crypto/sha1_ssse3_asm.S @@ -96,7 +96,7 @@ # cleanup workspace mov $8, %ecx mov %rsp, %rdi - xor %rax, %rax + xor %eax, %eax rep stosq mov %rbp, %rsp # deallocate workspace diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index c371bfee137a..2767c625a52c 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -65,7 +65,7 @@ # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF #else # define preempt_stop(clobbers) -# define resume_kernel restore_all +# define resume_kernel restore_all_kernel #endif .macro TRACE_IRQS_IRET @@ -77,6 +77,8 @@ #endif .endm +#define PTI_SWITCH_MASK (1 << PAGE_SHIFT) + /* * User gs save/restore * @@ -154,7 +156,52 @@ #endif /* CONFIG_X86_32_LAZY_GS */ -.macro SAVE_ALL pt_regs_ax=%eax +/* Unconditionally switch to user cr3 */ +.macro SWITCH_TO_USER_CR3 scratch_reg:req + ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI + + movl %cr3, \scratch_reg + orl $PTI_SWITCH_MASK, \scratch_reg + movl \scratch_reg, %cr3 +.Lend_\@: +.endm + +.macro BUG_IF_WRONG_CR3 no_user_check=0 +#ifdef CONFIG_DEBUG_ENTRY + ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI + .if \no_user_check == 0 + /* coming from usermode? */ + testl $SEGMENT_RPL_MASK, PT_CS(%esp) + jz .Lend_\@ + .endif + /* On user-cr3? */ + movl %cr3, %eax + testl $PTI_SWITCH_MASK, %eax + jnz .Lend_\@ + /* From userspace with kernel cr3 - BUG */ + ud2 +.Lend_\@: +#endif +.endm + +/* + * Switch to kernel cr3 if not already loaded and return current cr3 in + * \scratch_reg + */ +.macro SWITCH_TO_KERNEL_CR3 scratch_reg:req + ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI + movl %cr3, \scratch_reg + /* Test if we are already on kernel CR3 */ + testl $PTI_SWITCH_MASK, \scratch_reg + jz .Lend_\@ + andl $(~PTI_SWITCH_MASK), \scratch_reg + movl \scratch_reg, %cr3 + /* Return original CR3 in \scratch_reg */ + orl $PTI_SWITCH_MASK, \scratch_reg +.Lend_\@: +.endm + +.macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 cld PUSH_GS pushl %fs @@ -173,6 +220,29 @@ movl $(__KERNEL_PERCPU), %edx movl %edx, %fs SET_KERNEL_GS %edx + + /* Switch to kernel stack if necessary */ +.if \switch_stacks > 0 + SWITCH_TO_KERNEL_STACK +.endif + +.endm + +.macro SAVE_ALL_NMI cr3_reg:req + SAVE_ALL + + BUG_IF_WRONG_CR3 + + /* + * Now switch the CR3 when PTI is enabled. + * + * We can enter with either user or kernel cr3, the code will + * store the old cr3 in \cr3_reg and switches to the kernel cr3 + * if necessary. + */ + SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg + +.Lend_\@: .endm /* @@ -221,6 +291,349 @@ POP_GS_EX .endm +.macro RESTORE_ALL_NMI cr3_reg:req pop=0 + /* + * Now switch the CR3 when PTI is enabled. + * + * We enter with kernel cr3 and switch the cr3 to the value + * stored on \cr3_reg, which is either a user or a kernel cr3. + */ + ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI + + testl $PTI_SWITCH_MASK, \cr3_reg + jz .Lswitched_\@ + + /* User cr3 in \cr3_reg - write it to hardware cr3 */ + movl \cr3_reg, %cr3 + +.Lswitched_\@: + + BUG_IF_WRONG_CR3 + + RESTORE_REGS pop=\pop +.endm + +.macro CHECK_AND_APPLY_ESPFIX +#ifdef CONFIG_X86_ESPFIX32 +#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) + + ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_ESPFIX + + movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS + /* + * Warning: PT_OLDSS(%esp) contains the wrong/random values if we + * are returning to the kernel. + * See comments in process.c:copy_thread() for details. + */ + movb PT_OLDSS(%esp), %ah + movb PT_CS(%esp), %al + andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax + cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax + jne .Lend_\@ # returning to user-space with LDT SS + + /* + * Setup and switch to ESPFIX stack + * + * We're returning to userspace with a 16 bit stack. The CPU will not + * restore the high word of ESP for us on executing iret... This is an + * "official" bug of all the x86-compatible CPUs, which we can work + * around to make dosemu and wine happy. We do this by preloading the + * high word of ESP with the high word of the userspace ESP while + * compensating for the offset by changing to the ESPFIX segment with + * a base address that matches for the difference. + */ + mov %esp, %edx /* load kernel esp */ + mov PT_OLDESP(%esp), %eax /* load userspace esp */ + mov %dx, %ax /* eax: new kernel esp */ + sub %eax, %edx /* offset (low word is 0) */ + shr $16, %edx + mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ + mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ + pushl $__ESPFIX_SS + pushl %eax /* new kernel esp */ + /* + * Disable interrupts, but do not irqtrace this section: we + * will soon execute iret and the tracer was already set to + * the irqstate after the IRET: + */ + DISABLE_INTERRUPTS(CLBR_ANY) + lss (%esp), %esp /* switch to espfix segment */ +.Lend_\@: +#endif /* CONFIG_X86_ESPFIX32 */ +.endm + +/* + * Called with pt_regs fully populated and kernel segments loaded, + * so we can access PER_CPU and use the integer registers. + * + * We need to be very careful here with the %esp switch, because an NMI + * can happen everywhere. If the NMI handler finds itself on the + * entry-stack, it will overwrite the task-stack and everything we + * copied there. So allocate the stack-frame on the task-stack and + * switch to it before we do any copying. + */ + +#define CS_FROM_ENTRY_STACK (1 << 31) +#define CS_FROM_USER_CR3 (1 << 30) + +.macro SWITCH_TO_KERNEL_STACK + + ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV + + BUG_IF_WRONG_CR3 + + SWITCH_TO_KERNEL_CR3 scratch_reg=%eax + + /* + * %eax now contains the entry cr3 and we carry it forward in + * that register for the time this macro runs + */ + + /* Are we on the entry stack? Bail out if not! */ + movl PER_CPU_VAR(cpu_entry_area), %ecx + addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx + subl %esp, %ecx /* ecx = (end of entry_stack) - esp */ + cmpl $SIZEOF_entry_stack, %ecx + jae .Lend_\@ + + /* Load stack pointer into %esi and %edi */ + movl %esp, %esi + movl %esi, %edi + + /* Move %edi to the top of the entry stack */ + andl $(MASK_entry_stack), %edi + addl $(SIZEOF_entry_stack), %edi + + /* Load top of task-stack into %edi */ + movl TSS_entry2task_stack(%edi), %edi + + /* + * Clear unused upper bits of the dword containing the word-sized CS + * slot in pt_regs in case hardware didn't clear it for us. + */ + andl $(0x0000ffff), PT_CS(%esp) + + /* Special case - entry from kernel mode via entry stack */ +#ifdef CONFIG_VM86 + movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS + movb PT_CS(%esp), %cl + andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx +#else + movl PT_CS(%esp), %ecx + andl $SEGMENT_RPL_MASK, %ecx +#endif + cmpl $USER_RPL, %ecx + jb .Lentry_from_kernel_\@ + + /* Bytes to copy */ + movl $PTREGS_SIZE, %ecx + +#ifdef CONFIG_VM86 + testl $X86_EFLAGS_VM, PT_EFLAGS(%esi) + jz .Lcopy_pt_regs_\@ + + /* + * Stack-frame contains 4 additional segment registers when + * coming from VM86 mode + */ + addl $(4 * 4), %ecx + +#endif +.Lcopy_pt_regs_\@: + + /* Allocate frame on task-stack */ + subl %ecx, %edi + + /* Switch to task-stack */ + movl %edi, %esp + + /* + * We are now on the task-stack and can safely copy over the + * stack-frame + */ + shrl $2, %ecx + cld + rep movsl + + jmp .Lend_\@ + +.Lentry_from_kernel_\@: + + /* + * This handles the case when we enter the kernel from + * kernel-mode and %esp points to the entry-stack. When this + * happens we need to switch to the task-stack to run C code, + * but switch back to the entry-stack again when we approach + * iret and return to the interrupted code-path. This usually + * happens when we hit an exception while restoring user-space + * segment registers on the way back to user-space or when the + * sysenter handler runs with eflags.tf set. + * + * When we switch to the task-stack here, we can't trust the + * contents of the entry-stack anymore, as the exception handler + * might be scheduled out or moved to another CPU. Therefore we + * copy the complete entry-stack to the task-stack and set a + * marker in the iret-frame (bit 31 of the CS dword) to detect + * what we've done on the iret path. + * + * On the iret path we copy everything back and switch to the + * entry-stack, so that the interrupted kernel code-path + * continues on the same stack it was interrupted with. + * + * Be aware that an NMI can happen anytime in this code. + * + * %esi: Entry-Stack pointer (same as %esp) + * %edi: Top of the task stack + * %eax: CR3 on kernel entry + */ + + /* Calculate number of bytes on the entry stack in %ecx */ + movl %esi, %ecx + + /* %ecx to the top of entry-stack */ + andl $(MASK_entry_stack), %ecx + addl $(SIZEOF_entry_stack), %ecx + + /* Number of bytes on the entry stack to %ecx */ + sub %esi, %ecx + + /* Mark stackframe as coming from entry stack */ + orl $CS_FROM_ENTRY_STACK, PT_CS(%esp) + + /* + * Test the cr3 used to enter the kernel and add a marker + * so that we can switch back to it before iret. + */ + testl $PTI_SWITCH_MASK, %eax + jz .Lcopy_pt_regs_\@ + orl $CS_FROM_USER_CR3, PT_CS(%esp) + + /* + * %esi and %edi are unchanged, %ecx contains the number of + * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate + * the stack-frame on task-stack and copy everything over + */ + jmp .Lcopy_pt_regs_\@ + +.Lend_\@: +.endm + +/* + * Switch back from the kernel stack to the entry stack. + * + * The %esp register must point to pt_regs on the task stack. It will + * first calculate the size of the stack-frame to copy, depending on + * whether we return to VM86 mode or not. With that it uses 'rep movsl' + * to copy the contents of the stack over to the entry stack. + * + * We must be very careful here, as we can't trust the contents of the + * task-stack once we switched to the entry-stack. When an NMI happens + * while on the entry-stack, the NMI handler will switch back to the top + * of the task stack, overwriting our stack-frame we are about to copy. + * Therefore we switch the stack only after everything is copied over. + */ +.macro SWITCH_TO_ENTRY_STACK + + ALTERNATIVE "", "jmp .Lend_\@", X86_FEATURE_XENPV + + /* Bytes to copy */ + movl $PTREGS_SIZE, %ecx + +#ifdef CONFIG_VM86 + testl $(X86_EFLAGS_VM), PT_EFLAGS(%esp) + jz .Lcopy_pt_regs_\@ + + /* Additional 4 registers to copy when returning to VM86 mode */ + addl $(4 * 4), %ecx + +.Lcopy_pt_regs_\@: +#endif + + /* Initialize source and destination for movsl */ + movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi + subl %ecx, %edi + movl %esp, %esi + + /* Save future stack pointer in %ebx */ + movl %edi, %ebx + + /* Copy over the stack-frame */ + shrl $2, %ecx + cld + rep movsl + + /* + * Switch to entry-stack - needs to happen after everything is + * copied because the NMI handler will overwrite the task-stack + * when on entry-stack + */ + movl %ebx, %esp + +.Lend_\@: +.endm + +/* + * This macro handles the case when we return to kernel-mode on the iret + * path and have to switch back to the entry stack and/or user-cr3 + * + * See the comments below the .Lentry_from_kernel_\@ label in the + * SWITCH_TO_KERNEL_STACK macro for more details. + */ +.macro PARANOID_EXIT_TO_KERNEL_MODE + + /* + * Test if we entered the kernel with the entry-stack. Most + * likely we did not, because this code only runs on the + * return-to-kernel path. + */ + testl $CS_FROM_ENTRY_STACK, PT_CS(%esp) + jz .Lend_\@ + + /* Unlikely slow-path */ + + /* Clear marker from stack-frame */ + andl $(~CS_FROM_ENTRY_STACK), PT_CS(%esp) + + /* Copy the remaining task-stack contents to entry-stack */ + movl %esp, %esi + movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi + + /* Bytes on the task-stack to ecx */ + movl PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx + subl %esi, %ecx + + /* Allocate stack-frame on entry-stack */ + subl %ecx, %edi + + /* + * Save future stack-pointer, we must not switch until the + * copy is done, otherwise the NMI handler could destroy the + * contents of the task-stack we are about to copy. + */ + movl %edi, %ebx + + /* Do the copy */ + shrl $2, %ecx + cld + rep movsl + + /* Safe to switch to entry-stack now */ + movl %ebx, %esp + + /* + * We came from entry-stack and need to check if we also need to + * switch back to user cr3. + */ + testl $CS_FROM_USER_CR3, PT_CS(%esp) + jz .Lend_\@ + + /* Clear marker from stack-frame */ + andl $(~CS_FROM_USER_CR3), PT_CS(%esp) + + SWITCH_TO_USER_CR3 scratch_reg=%eax + +.Lend_\@: +.endm /* * %eax: prev task * %edx: next task @@ -351,9 +764,9 @@ ENTRY(resume_kernel) DISABLE_INTERRUPTS(CLBR_ANY) .Lneed_resched: cmpl $0, PER_CPU_VAR(__preempt_count) - jnz restore_all + jnz restore_all_kernel testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? - jz restore_all + jz restore_all_kernel call preempt_schedule_irq jmp .Lneed_resched END(resume_kernel) @@ -412,7 +825,21 @@ ENTRY(xen_sysenter_target) * 0(%ebp) arg6 */ ENTRY(entry_SYSENTER_32) - movl TSS_sysenter_sp0(%esp), %esp + /* + * On entry-stack with all userspace-regs live - save and + * restore eflags and %eax to use it as scratch-reg for the cr3 + * switch. + */ + pushfl + pushl %eax + BUG_IF_WRONG_CR3 no_user_check=1 + SWITCH_TO_KERNEL_CR3 scratch_reg=%eax + popl %eax + popfl + + /* Stack empty again, switch to task stack */ + movl TSS_entry2task_stack(%esp), %esp + .Lsysenter_past_esp: pushl $__USER_DS /* pt_regs->ss */ pushl %ebp /* pt_regs->sp (stashed in bp) */ @@ -421,7 +848,7 @@ ENTRY(entry_SYSENTER_32) pushl $__USER_CS /* pt_regs->cs */ pushl $0 /* pt_regs->ip = 0 (placeholder) */ pushl %eax /* pt_regs->orig_ax */ - SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ + SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest, stack already switched */ /* * SYSENTER doesn't filter flags, so we need to clear NT, AC @@ -460,25 +887,49 @@ ENTRY(entry_SYSENTER_32) /* Opportunistic SYSEXIT */ TRACE_IRQS_ON /* User mode traces as IRQs on. */ + + /* + * Setup entry stack - we keep the pointer in %eax and do the + * switch after almost all user-state is restored. + */ + + /* Load entry stack pointer and allocate frame for eflags/eax */ + movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax + subl $(2*4), %eax + + /* Copy eflags and eax to entry stack */ + movl PT_EFLAGS(%esp), %edi + movl PT_EAX(%esp), %esi + movl %edi, (%eax) + movl %esi, 4(%eax) + + /* Restore user registers and segments */ movl PT_EIP(%esp), %edx /* pt_regs->ip */ movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 1: mov PT_FS(%esp), %fs PTGS_TO_GS + popl %ebx /* pt_regs->bx */ addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ popl %esi /* pt_regs->si */ popl %edi /* pt_regs->di */ popl %ebp /* pt_regs->bp */ - popl %eax /* pt_regs->ax */ + + /* Switch to entry stack */ + movl %eax, %esp + + /* Now ready to switch the cr3 */ + SWITCH_TO_USER_CR3 scratch_reg=%eax /* * Restore all flags except IF. (We restore IF separately because * STI gives a one-instruction window in which we won't be interrupted, * whereas POPF does not.) */ - addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */ btrl $X86_EFLAGS_IF_BIT, (%esp) + BUG_IF_WRONG_CR3 no_user_check=1 popfl + popl %eax /* * Return back to the vDSO, which will pop ecx and edx. @@ -532,7 +983,8 @@ ENDPROC(entry_SYSENTER_32) ENTRY(entry_INT80_32) ASM_CLAC pushl %eax /* pt_regs->orig_ax */ - SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ + + SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */ /* * User mode is traced as though IRQs are on, and the interrupt gate @@ -546,24 +998,17 @@ ENTRY(entry_INT80_32) restore_all: TRACE_IRQS_IRET + SWITCH_TO_ENTRY_STACK .Lrestore_all_notrace: -#ifdef CONFIG_X86_ESPFIX32 - ALTERNATIVE "jmp .Lrestore_nocheck", "", X86_BUG_ESPFIX - - movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS - /* - * Warning: PT_OLDSS(%esp) contains the wrong/random values if we - * are returning to the kernel. - * See comments in process.c:copy_thread() for details. - */ - movb PT_OLDSS(%esp), %ah - movb PT_CS(%esp), %al - andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax - cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax - je .Lldt_ss # returning to user-space with LDT SS -#endif + CHECK_AND_APPLY_ESPFIX .Lrestore_nocheck: - RESTORE_REGS 4 # skip orig_eax/error_code + /* Switch back to user CR3 */ + SWITCH_TO_USER_CR3 scratch_reg=%eax + + BUG_IF_WRONG_CR3 + + /* Restore user state */ + RESTORE_REGS pop=4 # skip orig_eax/error_code .Lirq_return: /* * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization @@ -572,46 +1017,33 @@ restore_all: */ INTERRUPT_RETURN +restore_all_kernel: + TRACE_IRQS_IRET + PARANOID_EXIT_TO_KERNEL_MODE + BUG_IF_WRONG_CR3 + RESTORE_REGS 4 + jmp .Lirq_return + .section .fixup, "ax" ENTRY(iret_exc ) pushl $0 # no error code pushl $do_iret_error - jmp common_exception -.previous - _ASM_EXTABLE(.Lirq_return, iret_exc) -#ifdef CONFIG_X86_ESPFIX32 -.Lldt_ss: -/* - * Setup and switch to ESPFIX stack - * - * We're returning to userspace with a 16 bit stack. The CPU will not - * restore the high word of ESP for us on executing iret... This is an - * "official" bug of all the x86-compatible CPUs, which we can work - * around to make dosemu and wine happy. We do this by preloading the - * high word of ESP with the high word of the userspace ESP while - * compensating for the offset by changing to the ESPFIX segment with - * a base address that matches for the difference. - */ -#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) - mov %esp, %edx /* load kernel esp */ - mov PT_OLDESP(%esp), %eax /* load userspace esp */ - mov %dx, %ax /* eax: new kernel esp */ - sub %eax, %edx /* offset (low word is 0) */ - shr $16, %edx - mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ - mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ - pushl $__ESPFIX_SS - pushl %eax /* new kernel esp */ +#ifdef CONFIG_DEBUG_ENTRY /* - * Disable interrupts, but do not irqtrace this section: we - * will soon execute iret and the tracer was already set to - * the irqstate after the IRET: + * The stack-frame here is the one that iret faulted on, so its a + * return-to-user frame. We are on kernel-cr3 because we come here from + * the fixup code. This confuses the CR3 checker, so switch to user-cr3 + * as the checker expects it. */ - DISABLE_INTERRUPTS(CLBR_ANY) - lss (%esp), %esp /* switch to espfix segment */ - jmp .Lrestore_nocheck + pushl %eax + SWITCH_TO_USER_CR3 scratch_reg=%eax + popl %eax #endif + + jmp common_exception +.previous + _ASM_EXTABLE(.Lirq_return, iret_exc) ENDPROC(entry_INT80_32) .macro FIXUP_ESPFIX_STACK @@ -671,7 +1103,8 @@ END(irq_entries_start) common_interrupt: ASM_CLAC addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ - SAVE_ALL + + SAVE_ALL switch_stacks=1 ENCODE_FRAME_POINTER TRACE_IRQS_OFF movl %esp, %eax @@ -679,16 +1112,16 @@ common_interrupt: jmp ret_from_intr ENDPROC(common_interrupt) -#define BUILD_INTERRUPT3(name, nr, fn) \ -ENTRY(name) \ - ASM_CLAC; \ - pushl $~(nr); \ - SAVE_ALL; \ - ENCODE_FRAME_POINTER; \ - TRACE_IRQS_OFF \ - movl %esp, %eax; \ - call fn; \ - jmp ret_from_intr; \ +#define BUILD_INTERRUPT3(name, nr, fn) \ +ENTRY(name) \ + ASM_CLAC; \ + pushl $~(nr); \ + SAVE_ALL switch_stacks=1; \ + ENCODE_FRAME_POINTER; \ + TRACE_IRQS_OFF \ + movl %esp, %eax; \ + call fn; \ + jmp ret_from_intr; \ ENDPROC(name) #define BUILD_INTERRUPT(name, nr) \ @@ -920,16 +1353,20 @@ common_exception: pushl %es pushl %ds pushl %eax + movl $(__USER_DS), %eax + movl %eax, %ds + movl %eax, %es + movl $(__KERNEL_PERCPU), %eax + movl %eax, %fs pushl %ebp pushl %edi pushl %esi pushl %edx pushl %ecx pushl %ebx + SWITCH_TO_KERNEL_STACK ENCODE_FRAME_POINTER cld - movl $(__KERNEL_PERCPU), %ecx - movl %ecx, %fs UNWIND_ESPFIX_STACK GS_TO_REG %ecx movl PT_GS(%esp), %edi # get the function address @@ -937,9 +1374,6 @@ common_exception: movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart REG_TO_PTGS %ecx SET_KERNEL_GS %ecx - movl $(__USER_DS), %ecx - movl %ecx, %ds - movl %ecx, %es TRACE_IRQS_OFF movl %esp, %eax # pt_regs pointer CALL_NOSPEC %edi @@ -948,40 +1382,12 @@ END(common_exception) ENTRY(debug) /* - * #DB can happen at the first instruction of - * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this - * happens, then we will be running on a very small stack. We - * need to detect this condition and switch to the thread - * stack before calling any C code at all. - * - * If you edit this code, keep in mind that NMIs can happen in here. + * Entry from sysenter is now handled in common_exception */ ASM_CLAC pushl $-1 # mark this as an int - SAVE_ALL - ENCODE_FRAME_POINTER - xorl %edx, %edx # error code 0 - movl %esp, %eax # pt_regs pointer - - /* Are we currently on the SYSENTER stack? */ - movl PER_CPU_VAR(cpu_entry_area), %ecx - addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx - subl %eax, %ecx /* ecx = (end of entry_stack) - esp */ - cmpl $SIZEOF_entry_stack, %ecx - jb .Ldebug_from_sysenter_stack - - TRACE_IRQS_OFF - call do_debug - jmp ret_from_exception - -.Ldebug_from_sysenter_stack: - /* We're on the SYSENTER stack. Switch off. */ - movl %esp, %ebx - movl PER_CPU_VAR(cpu_current_top_of_stack), %esp - TRACE_IRQS_OFF - call do_debug - movl %ebx, %esp - jmp ret_from_exception + pushl $do_debug + jmp common_exception END(debug) /* @@ -993,6 +1399,7 @@ END(debug) */ ENTRY(nmi) ASM_CLAC + #ifdef CONFIG_X86_ESPFIX32 pushl %eax movl %ss, %eax @@ -1002,7 +1409,7 @@ ENTRY(nmi) #endif pushl %eax # pt_regs->orig_ax - SAVE_ALL + SAVE_ALL_NMI cr3_reg=%edi ENCODE_FRAME_POINTER xorl %edx, %edx # zero error code movl %esp, %eax # pt_regs pointer @@ -1016,7 +1423,7 @@ ENTRY(nmi) /* Not on SYSENTER stack. */ call do_nmi - jmp .Lrestore_all_notrace + jmp .Lnmi_return .Lnmi_from_sysenter_stack: /* @@ -1027,7 +1434,11 @@ ENTRY(nmi) movl PER_CPU_VAR(cpu_current_top_of_stack), %esp call do_nmi movl %ebx, %esp - jmp .Lrestore_all_notrace + +.Lnmi_return: + CHECK_AND_APPLY_ESPFIX + RESTORE_ALL_NMI cr3_reg=%edi pop=4 + jmp .Lirq_return #ifdef CONFIG_X86_ESPFIX32 .Lnmi_espfix_stack: @@ -1042,12 +1453,12 @@ ENTRY(nmi) pushl 16(%esp) .endr pushl %eax - SAVE_ALL + SAVE_ALL_NMI cr3_reg=%edi ENCODE_FRAME_POINTER FIXUP_ESPFIX_STACK # %eax == %esp xorl %edx, %edx # zero error code call do_nmi - RESTORE_REGS + RESTORE_ALL_NMI cr3_reg=%edi lss 12+4(%esp), %esp # back to espfix stack jmp .Lirq_return #endif @@ -1056,7 +1467,8 @@ END(nmi) ENTRY(int3) ASM_CLAC pushl $-1 # mark this as an int - SAVE_ALL + + SAVE_ALL switch_stacks=1 ENCODE_FRAME_POINTER TRACE_IRQS_OFF xorl %edx, %edx # zero error code diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 73a522d53b53..957dfb693ecc 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -92,7 +92,7 @@ END(native_usergs_sysret64) .endm .macro TRACE_IRQS_IRETQ_DEBUG - bt $9, EFLAGS(%rsp) /* interrupts off? */ + btl $9, EFLAGS(%rsp) /* interrupts off? */ jnc 1f TRACE_IRQS_ON_DEBUG 1: @@ -408,6 +408,7 @@ ENTRY(ret_from_fork) 1: /* kernel thread */ + UNWIND_HINT_EMPTY movq %r12, %rdi CALL_NOSPEC %rbx /* @@ -701,7 +702,7 @@ retint_kernel: #ifdef CONFIG_PREEMPT /* Interrupts are off */ /* Check if we need preemption */ - bt $9, EFLAGS(%rsp) /* were interrupts off? */ + btl $9, EFLAGS(%rsp) /* were interrupts off? */ jnc 1f 0: cmpl $0, PER_CPU_VAR(__preempt_count) jnz 1f @@ -981,7 +982,7 @@ ENTRY(\sym) call \do_sym - jmp error_exit /* %ebx: no swapgs flag */ + jmp error_exit .endif END(\sym) .endm @@ -1222,7 +1223,6 @@ END(paranoid_exit) /* * Save all registers in pt_regs, and switch GS if needed. - * Return: EBX=0: came from user mode; EBX=1: otherwise */ ENTRY(error_entry) UNWIND_HINT_FUNC @@ -1269,7 +1269,6 @@ ENTRY(error_entry) * for these here too. */ .Lerror_kernelspace: - incl %ebx leaq native_irq_return_iret(%rip), %rcx cmpq %rcx, RIP+8(%rsp) je .Lerror_bad_iret @@ -1303,28 +1302,20 @@ ENTRY(error_entry) /* * Pretend that the exception came from user mode: set up pt_regs - * as if we faulted immediately after IRET and clear EBX so that - * error_exit knows that we will be returning to user mode. + * as if we faulted immediately after IRET. */ mov %rsp, %rdi call fixup_bad_iret mov %rax, %rsp - decl %ebx jmp .Lerror_entry_from_usermode_after_swapgs END(error_entry) - -/* - * On entry, EBX is a "return to kernel mode" flag: - * 1: already in kernel mode, don't need SWAPGS - * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode - */ ENTRY(error_exit) UNWIND_HINT_REGS DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_OFF - testl %ebx, %ebx - jnz retint_kernel + testb $3, CS(%rsp) + jz retint_kernel jmp retint_user END(error_exit) diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 261802b1cc50..9f695f517747 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -46,10 +46,8 @@ targets += $(vdso_img_sodbg) $(vdso_img-y:%=vdso%.so) CPPFLAGS_vdso.lds += -P -C -VDSO_LDFLAGS_vdso.lds = -m64 -Wl,-soname=linux-vdso.so.1 \ - -Wl,--no-undefined \ - -Wl,-z,max-page-size=4096 -Wl,-z,common-page-size=4096 \ - $(DISABLE_LTO) +VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined \ + -z max-page-size=4096 -z common-page-size=4096 $(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE $(call if_changed,vdso) @@ -58,9 +56,7 @@ HOST_EXTRACFLAGS += -I$(srctree)/tools/include -I$(srctree)/include/uapi -I$(src hostprogs-y += vdso2c quiet_cmd_vdso2c = VDSO2C $@ -define cmd_vdso2c - $(obj)/vdso2c $< $(<:%.dbg=%) $@ -endef + cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@ $(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE $(call if_changed,vdso2c) @@ -95,10 +91,8 @@ CFLAGS_REMOVE_vvar.o = -pg # CPPFLAGS_vdsox32.lds = $(CPPFLAGS_vdso.lds) -VDSO_LDFLAGS_vdsox32.lds = -Wl,-m,elf32_x86_64 \ - -Wl,-soname=linux-vdso.so.1 \ - -Wl,-z,max-page-size=4096 \ - -Wl,-z,common-page-size=4096 +VDSO_LDFLAGS_vdsox32.lds = -m elf32_x86_64 -soname linux-vdso.so.1 \ + -z max-page-size=4096 -z common-page-size=4096 # x32-rebranded versions vobjx32s-y := $(vobjs-y:.o=-x32.o) @@ -123,7 +117,7 @@ $(obj)/vdsox32.so.dbg: $(obj)/vdsox32.lds $(vobjx32s) FORCE $(call if_changed,vdso) CPPFLAGS_vdso32.lds = $(CPPFLAGS_vdso.lds) -VDSO_LDFLAGS_vdso32.lds = -m32 -Wl,-m,elf_i386 -Wl,-soname=linux-gate.so.1 +VDSO_LDFLAGS_vdso32.lds = -m elf_i386 -soname linux-gate.so.1 targets += vdso32/vdso32.lds targets += vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o @@ -157,13 +151,13 @@ $(obj)/vdso32.so.dbg: FORCE \ # The DSO images are built using a special linker script. # quiet_cmd_vdso = VDSO $@ - cmd_vdso = $(CC) -nostdlib -o $@ \ + cmd_vdso = $(LD) -nostdlib -o $@ \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ - -Wl,-T,$(filter %.lds,$^) $(filter %.o,$^) && \ + -T $(filter %.lds,$^) $(filter %.o,$^) && \ sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@' -VDSO_LDFLAGS = -fPIC -shared $(call cc-ldoption, -Wl$(comma)--hash-style=both) \ - $(call cc-ldoption, -Wl$(comma)--build-id) -Wl,-Bsymbolic $(LTO_CFLAGS) +VDSO_LDFLAGS = -shared $(call ld-option, --hash-style=both) \ + $(call ld-option, --build-id) -Bsymbolic GCOV_PROFILE := n # diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 4b98101209a1..d50bb4dc0650 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -579,7 +579,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) { struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); struct perf_event *event = pcpu->event; - struct hw_perf_event *hwc = &event->hw; + struct hw_perf_event *hwc; struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; @@ -602,6 +602,10 @@ fail: return 0; } + if (WARN_ON_ONCE(!event)) + goto fail; + + hwc = &event->hw; msr = hwc->config_base; buf = ibs_data.regs; rdmsrl(msr, *buf); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 707b2a96e516..035c37481f57 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2041,15 +2041,15 @@ static void intel_pmu_disable_event(struct perf_event *event) cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx); cpuc->intel_cp_status &= ~(1ull << hwc->idx); + if (unlikely(event->attr.precise_ip)) + intel_pmu_pebs_disable(event); + if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { intel_pmu_disable_fixed(hwc); return; } x86_pmu_disable_event(event); - - if (unlikely(event->attr.precise_ip)) - intel_pmu_pebs_disable(event); } static void intel_pmu_del_event(struct perf_event *event) @@ -2068,17 +2068,19 @@ static void intel_pmu_read_event(struct perf_event *event) x86_perf_event_update(event); } -static void intel_pmu_enable_fixed(struct hw_perf_event *hwc) +static void intel_pmu_enable_fixed(struct perf_event *event) { + struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx - INTEL_PMC_IDX_FIXED; - u64 ctrl_val, bits, mask; + u64 ctrl_val, mask, bits = 0; /* - * Enable IRQ generation (0x8), + * Enable IRQ generation (0x8), if not PEBS, * and enable ring-3 counting (0x2) and ring-0 counting (0x1) * if requested: */ - bits = 0x8ULL; + if (!event->attr.precise_ip) + bits |= 0x8; if (hwc->config & ARCH_PERFMON_EVENTSEL_USR) bits |= 0x2; if (hwc->config & ARCH_PERFMON_EVENTSEL_OS) @@ -2120,14 +2122,14 @@ static void intel_pmu_enable_event(struct perf_event *event) if (unlikely(event_is_checkpointed(event))) cpuc->intel_cp_status |= (1ull << hwc->idx); + if (unlikely(event->attr.precise_ip)) + intel_pmu_pebs_enable(event); + if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { - intel_pmu_enable_fixed(hwc); + intel_pmu_enable_fixed(event); return; } - if (unlikely(event->attr.precise_ip)) - intel_pmu_pebs_enable(event); - __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); } @@ -2280,7 +2282,10 @@ again: * counters from the GLOBAL_STATUS mask and we always process PEBS * events via drain_pebs(). */ - status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK); + if (x86_pmu.flags & PMU_FL_PEBS_ALL) + status &= ~cpuc->pebs_enabled; + else + status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK); /* * PEBS overflow sets bit 62 in the global status register @@ -2997,6 +3002,9 @@ static int intel_pmu_hw_config(struct perf_event *event) } if (x86_pmu.pebs_aliases) x86_pmu.pebs_aliases(event); + + if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) + event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; } if (needs_branch_stack(event)) { @@ -4069,7 +4077,6 @@ __init int intel_pmu_init(void) intel_pmu_lbr_init_skl(); x86_pmu.event_constraints = intel_slm_event_constraints; - x86_pmu.pebs_constraints = intel_glp_pebs_event_constraints; x86_pmu.extra_regs = intel_glm_extra_regs; /* * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS @@ -4079,6 +4086,7 @@ __init int intel_pmu_init(void) x86_pmu.pebs_prec_dist = true; x86_pmu.lbr_pt_coexist = true; x86_pmu.flags |= PMU_FL_HAS_RSP_1; + x86_pmu.flags |= PMU_FL_PEBS_ALL; x86_pmu.get_event_constraints = glp_get_event_constraints; x86_pmu.cpu_events = glm_events_attrs; /* Goldmont Plus has 4-wide pipeline */ diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 8a10a045b57b..b7b01d762d32 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -408,9 +408,11 @@ static int alloc_bts_buffer(int cpu) ds->bts_buffer_base = (unsigned long) cea; ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL); ds->bts_index = ds->bts_buffer_base; - max = BTS_RECORD_SIZE * (BTS_BUFFER_SIZE / BTS_RECORD_SIZE); - ds->bts_absolute_maximum = ds->bts_buffer_base + max; - ds->bts_interrupt_threshold = ds->bts_absolute_maximum - (max / 16); + max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE; + ds->bts_absolute_maximum = ds->bts_buffer_base + + max * BTS_RECORD_SIZE; + ds->bts_interrupt_threshold = ds->bts_absolute_maximum - + (max / 16) * BTS_RECORD_SIZE; return 0; } @@ -711,12 +713,6 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { EVENT_CONSTRAINT_END }; -struct event_constraint intel_glp_pebs_event_constraints[] = { - /* Allow all events as PEBS with no flags */ - INTEL_ALL_EVENT_CONSTRAINT(0, 0xf), - EVENT_CONSTRAINT_END -}; - struct event_constraint intel_nehalem_pebs_event_constraints[] = { INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */ INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */ @@ -869,6 +865,13 @@ struct event_constraint *intel_pebs_constraints(struct perf_event *event) } } + /* + * Extended PEBS support + * Makes the PEBS code search the normal constraints. + */ + if (x86_pmu.flags & PMU_FL_PEBS_ALL) + return NULL; + return &emptyconstraint; } @@ -894,10 +897,16 @@ static inline void pebs_update_threshold(struct cpu_hw_events *cpuc) { struct debug_store *ds = cpuc->ds; u64 threshold; + int reserved; + + if (x86_pmu.flags & PMU_FL_PEBS_ALL) + reserved = x86_pmu.max_pebs_events + x86_pmu.num_counters_fixed; + else + reserved = x86_pmu.max_pebs_events; if (cpuc->n_pebs == cpuc->n_large_pebs) { threshold = ds->pebs_absolute_maximum - - x86_pmu.max_pebs_events * x86_pmu.pebs_record_size; + reserved * x86_pmu.pebs_record_size; } else { threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size; } @@ -961,7 +970,11 @@ void intel_pmu_pebs_enable(struct perf_event *event) * This must be done in pmu::start(), because PERF_EVENT_IOC_PERIOD. */ if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) { - ds->pebs_event_reset[hwc->idx] = + unsigned int idx = hwc->idx; + + if (idx >= INTEL_PMC_IDX_FIXED) + idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED); + ds->pebs_event_reset[idx] = (u64)(-hwc->sample_period) & x86_pmu.cntval_mask; } else { ds->pebs_event_reset[hwc->idx] = 0; @@ -1184,16 +1197,20 @@ static void setup_pebs_sample_data(struct perf_event *event, } /* + * We must however always use iregs for the unwinder to stay sane; the + * record BP,SP,IP can point into thin air when the record is from a + * previous PMI context or an (I)RET happend between the record and + * PMI. + */ + if (sample_type & PERF_SAMPLE_CALLCHAIN) + data->callchain = perf_callchain(event, iregs); + + /* * We use the interrupt regs as a base because the PEBS record does not * contain a full regs set, specifically it seems to lack segment * descriptors, which get used by things like user_mode(). * * In the simple case fix up only the IP for PERF_SAMPLE_IP. - * - * We must however always use BP,SP from iregs for the unwinder to stay - * sane; the record BP,SP can point into thin air when the record is - * from a previous PMI context or an (I)RET happend between the record - * and PMI. */ *regs = *iregs; @@ -1212,15 +1229,8 @@ static void setup_pebs_sample_data(struct perf_event *event, regs->si = pebs->si; regs->di = pebs->di; - /* - * Per the above; only set BP,SP if we don't need callchains. - * - * XXX: does this make sense? - */ - if (!(sample_type & PERF_SAMPLE_CALLCHAIN)) { - regs->bp = pebs->bp; - regs->sp = pebs->sp; - } + regs->bp = pebs->bp; + regs->sp = pebs->sp; #ifndef CONFIG_X86_32 regs->r8 = pebs->r8; @@ -1482,9 +1492,10 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) struct debug_store *ds = cpuc->ds; struct perf_event *event; void *base, *at, *top; - short counts[MAX_PEBS_EVENTS] = {}; - short error[MAX_PEBS_EVENTS] = {}; - int bit, i; + short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; + short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; + int bit, i, size; + u64 mask; if (!x86_pmu.pebs_active) return; @@ -1494,6 +1505,13 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) ds->pebs_index = ds->pebs_buffer_base; + mask = (1ULL << x86_pmu.max_pebs_events) - 1; + size = x86_pmu.max_pebs_events; + if (x86_pmu.flags & PMU_FL_PEBS_ALL) { + mask |= ((1ULL << x86_pmu.num_counters_fixed) - 1) << INTEL_PMC_IDX_FIXED; + size = INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed; + } + if (unlikely(base >= top)) { /* * The drain_pebs() could be called twice in a short period @@ -1503,7 +1521,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) * update the event->count for this case. */ for_each_set_bit(bit, (unsigned long *)&cpuc->pebs_enabled, - x86_pmu.max_pebs_events) { + size) { event = cpuc->events[bit]; if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD) intel_pmu_save_and_restart_reload(event, 0); @@ -1516,12 +1534,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) u64 pebs_status; pebs_status = p->status & cpuc->pebs_enabled; - pebs_status &= (1ULL << x86_pmu.max_pebs_events) - 1; + pebs_status &= mask; /* PEBS v3 has more accurate status bits */ if (x86_pmu.intel_cap.pebs_format >= 3) { for_each_set_bit(bit, (unsigned long *)&pebs_status, - x86_pmu.max_pebs_events) + size) counts[bit]++; continue; @@ -1569,7 +1587,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs) counts[bit]++; } - for (bit = 0; bit < x86_pmu.max_pebs_events; bit++) { + for (bit = 0; bit < size; bit++) { if ((counts[bit] == 0) && (error[bit] == 0)) continue; diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cf372b90557e..f3e006bed9a7 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -216,6 +216,8 @@ static void intel_pmu_lbr_reset_64(void) void intel_pmu_lbr_reset(void) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + if (!x86_pmu.lbr_nr) return; @@ -223,6 +225,9 @@ void intel_pmu_lbr_reset(void) intel_pmu_lbr_reset_32(); else intel_pmu_lbr_reset_64(); + + cpuc->last_task_ctx = NULL; + cpuc->last_log_id = 0; } /* @@ -334,6 +339,7 @@ static inline u64 rdlbr_to(unsigned int idx) static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int i; unsigned lbr_idx, mask; u64 tos; @@ -344,9 +350,21 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) return; } - mask = x86_pmu.lbr_nr - 1; tos = task_ctx->tos; - for (i = 0; i < tos; i++) { + /* + * Does not restore the LBR registers, if + * - No one else touched them, and + * - Did not enter C6 + */ + if ((task_ctx == cpuc->last_task_ctx) && + (task_ctx->log_id == cpuc->last_log_id) && + rdlbr_from(tos)) { + task_ctx->lbr_stack_state = LBR_NONE; + return; + } + + mask = x86_pmu.lbr_nr - 1; + for (i = 0; i < task_ctx->valid_lbrs; i++) { lbr_idx = (tos - i) & mask; wrlbr_from(lbr_idx, task_ctx->lbr_from[i]); wrlbr_to (lbr_idx, task_ctx->lbr_to[i]); @@ -354,14 +372,24 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } + + for (; i < x86_pmu.lbr_nr; i++) { + lbr_idx = (tos - i) & mask; + wrlbr_from(lbr_idx, 0); + wrlbr_to(lbr_idx, 0); + if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) + wrmsrl(MSR_LBR_INFO_0 + lbr_idx, 0); + } + wrmsrl(x86_pmu.lbr_tos, tos); task_ctx->lbr_stack_state = LBR_NONE; } static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); unsigned lbr_idx, mask; - u64 tos; + u64 tos, from; int i; if (task_ctx->lbr_callstack_users == 0) { @@ -371,15 +399,22 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) mask = x86_pmu.lbr_nr - 1; tos = intel_pmu_lbr_tos(); - for (i = 0; i < tos; i++) { + for (i = 0; i < x86_pmu.lbr_nr; i++) { lbr_idx = (tos - i) & mask; - task_ctx->lbr_from[i] = rdlbr_from(lbr_idx); + from = rdlbr_from(lbr_idx); + if (!from) + break; + task_ctx->lbr_from[i] = from; task_ctx->lbr_to[i] = rdlbr_to(lbr_idx); if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); } + task_ctx->valid_lbrs = i; task_ctx->tos = tos; task_ctx->lbr_stack_state = LBR_VALID; + + cpuc->last_task_ctx = task_ctx; + cpuc->last_log_id = ++task_ctx->log_id; } void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in) @@ -531,7 +566,7 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) */ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) { - bool need_info = false; + bool need_info = false, call_stack = false; unsigned long mask = x86_pmu.lbr_nr - 1; int lbr_format = x86_pmu.intel_cap.lbr_format; u64 tos = intel_pmu_lbr_tos(); @@ -542,7 +577,7 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (cpuc->lbr_sel) { need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); if (cpuc->lbr_sel->config & LBR_CALL_STACK) - num = tos; + call_stack = true; } for (i = 0; i < num; i++) { @@ -555,6 +590,13 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) from = rdlbr_from(lbr_idx); to = rdlbr_to(lbr_idx); + /* + * Read LBR call stack entries + * until invalid entry (0s) is detected. + */ + if (call_stack && !from) + break; + if (lbr_format == LBR_FORMAT_INFO && need_info) { u64 info; diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c9e1e0bef3c3..e17ab885b1e9 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -28,7 +28,7 @@ #define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff) #define UNCORE_PCI_DEV_IDX(data) (data & 0xff) #define UNCORE_EXTRA_PCI_DEV 0xff -#define UNCORE_EXTRA_PCI_DEV_MAX 3 +#define UNCORE_EXTRA_PCI_DEV_MAX 4 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 87dc0263a2e1..51d7c117e3c7 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -1029,6 +1029,7 @@ void snbep_uncore_cpu_init(void) enum { SNBEP_PCI_QPI_PORT0_FILTER, SNBEP_PCI_QPI_PORT1_FILTER, + BDX_PCI_QPI_PORT2_FILTER, HSWEP_PCI_PCU_3, }; @@ -3286,15 +3287,18 @@ static const struct pci_device_id bdx_uncore_pci_ids[] = { }, { /* QPI Port 0 filter */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f86), - .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 0), + .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, + SNBEP_PCI_QPI_PORT0_FILTER), }, { /* QPI Port 1 filter */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f96), - .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 1), + .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, + SNBEP_PCI_QPI_PORT1_FILTER), }, { /* QPI Port 2 filter */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f46), - .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 2), + .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, + BDX_PCI_QPI_PORT2_FILTER), }, { /* PCU.3 (for Capability registers) */ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fc0), diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 9f3711470ec1..156286335351 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -163,6 +163,7 @@ struct intel_excl_cntrs { unsigned core_id; /* per-core: core id */ }; +struct x86_perf_task_context; #define MAX_LBR_ENTRIES 32 enum { @@ -214,6 +215,8 @@ struct cpu_hw_events { struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES]; struct er_account *lbr_sel; u64 br_sel; + struct x86_perf_task_context *last_task_ctx; + int last_log_id; /* * Intel host/guest exclude bits @@ -648,8 +651,10 @@ struct x86_perf_task_context { u64 lbr_to[MAX_LBR_ENTRIES]; u64 lbr_info[MAX_LBR_ENTRIES]; int tos; + int valid_lbrs; int lbr_callstack_users; int lbr_stack_state; + int log_id; }; #define x86_add_quirk(func_) \ @@ -668,6 +673,7 @@ do { \ #define PMU_FL_HAS_RSP_1 0x2 /* has 2 equivalent offcore_rsp regs */ #define PMU_FL_EXCL_CNTRS 0x4 /* has exclusive counter requirements */ #define PMU_FL_EXCL_ENABLED 0x8 /* exclusive counter active */ +#define PMU_FL_PEBS_ALL 0x10 /* all events are valid PEBS events */ #define EVENT_VAR(_id) event_attr_##_id #define EVENT_PTR(_id) &event_attr_##_id.attr.attr diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index f68855499391..5b0f613428c2 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -31,6 +31,8 @@ #include <asm/mshyperv.h> #include <asm/apic.h> +#include <asm/trace/hyperv.h> + static struct apic orig_apic; static u64 hv_apic_icr_read(void) @@ -99,6 +101,9 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) int nr_bank = 0; int ret = 1; + if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) + return false; + local_irq_save(flags); arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); @@ -114,6 +119,8 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K; nr_bank = cpumask_to_vpset(&(ipi_arg->vp_set), mask); } + if (nr_bank < 0) + goto ipi_mask_ex_done; if (!nr_bank) ipi_arg->vp_set.format = HV_GENERIC_SET_ALL; @@ -128,10 +135,10 @@ ipi_mask_ex_done: static bool __send_ipi_mask(const struct cpumask *mask, int vector) { int cur_cpu, vcpu; - struct ipi_arg_non_ex **arg; - struct ipi_arg_non_ex *ipi_arg; + struct ipi_arg_non_ex ipi_arg; int ret = 1; - unsigned long flags; + + trace_hyperv_send_ipi_mask(mask, vector); if (cpumask_empty(mask)) return true; @@ -142,37 +149,43 @@ static bool __send_ipi_mask(const struct cpumask *mask, int vector) if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) return false; - if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) - return __send_ipi_mask_ex(mask, vector); - - local_irq_save(flags); - arg = (struct ipi_arg_non_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); - - ipi_arg = *arg; - if (unlikely(!ipi_arg)) - goto ipi_mask_done; - - ipi_arg->vector = vector; - ipi_arg->reserved = 0; - ipi_arg->cpu_mask = 0; + /* + * From the supplied CPU set we need to figure out if we can get away + * with cheaper HVCALL_SEND_IPI hypercall. This is possible when the + * highest VP number in the set is < 64. As VP numbers are usually in + * ascending order and match Linux CPU ids, here is an optimization: + * we check the VP number for the highest bit in the supplied set first + * so we can quickly find out if using HVCALL_SEND_IPI_EX hypercall is + * a must. We will also check all VP numbers when walking the supplied + * CPU set to remain correct in all cases. + */ + if (hv_cpu_number_to_vp_number(cpumask_last(mask)) >= 64) + goto do_ex_hypercall; + + ipi_arg.vector = vector; + ipi_arg.cpu_mask = 0; for_each_cpu(cur_cpu, mask) { vcpu = hv_cpu_number_to_vp_number(cur_cpu); + if (vcpu == VP_INVAL) + return false; + /* * This particular version of the IPI hypercall can * only target upto 64 CPUs. */ if (vcpu >= 64) - goto ipi_mask_done; + goto do_ex_hypercall; - __set_bit(vcpu, (unsigned long *)&ipi_arg->cpu_mask); + __set_bit(vcpu, (unsigned long *)&ipi_arg.cpu_mask); } - ret = hv_do_hypercall(HVCALL_SEND_IPI, ipi_arg, NULL); - -ipi_mask_done: - local_irq_restore(flags); + ret = hv_do_fast_hypercall16(HVCALL_SEND_IPI, ipi_arg.vector, + ipi_arg.cpu_mask); return ((ret == 0) ? true : false); + +do_ex_hypercall: + return __send_ipi_mask_ex(mask, vector); } static bool __send_ipi_one(int cpu, int vector) @@ -228,10 +241,7 @@ static void hv_send_ipi_self(int vector) void __init hv_apic_init(void) { if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) { - if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) - pr_info("Hyper-V: Using ext hypercalls for IPI\n"); - else - pr_info("Hyper-V: Using IPI hypercalls\n"); + pr_info("Hyper-V: Using IPI hypercalls\n"); /* * Set the IPI entry points. */ diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 4c431e1c1eff..1ff420217298 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -265,7 +265,7 @@ void __init hyperv_init(void) { u64 guest_id, required_msrs; union hv_x64_msr_hypercall_contents hypercall_msr; - int cpuhp; + int cpuhp, i; if (x86_hyper_type != X86_HYPER_MS_HYPERV) return; @@ -293,6 +293,9 @@ void __init hyperv_init(void) if (!hv_vp_index) return; + for (i = 0; i < num_possible_cpus(); i++) + hv_vp_index[i] = VP_INVAL; + hv_vp_assist_page = kcalloc(num_possible_cpus(), sizeof(*hv_vp_assist_page), GFP_KERNEL); if (!hv_vp_assist_page) { diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c index de27615c51ea..1147e1fed7ff 100644 --- a/arch/x86/hyperv/mmu.c +++ b/arch/x86/hyperv/mmu.c @@ -16,6 +16,8 @@ /* Each gva in gva_list encodes up to 4096 pages to flush */ #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) +static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, + const struct flush_tlb_info *info); /* * Fills in gva_list starting from offset. Returns the number of items added. @@ -93,10 +95,29 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus, if (cpumask_equal(cpus, cpu_present_mask)) { flush->flags |= HV_FLUSH_ALL_PROCESSORS; } else { + /* + * From the supplied CPU set we need to figure out if we can get + * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE} + * hypercalls. This is possible when the highest VP number in + * the set is < 64. As VP numbers are usually in ascending order + * and match Linux CPU ids, here is an optimization: we check + * the VP number for the highest bit in the supplied set first + * so we can quickly find out if using *_EX hypercalls is a + * must. We will also check all VP numbers when walking the + * supplied CPU set to remain correct in all cases. + */ + if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64) + goto do_ex_hypercall; + for_each_cpu(cpu, cpus) { vcpu = hv_cpu_number_to_vp_number(cpu); - if (vcpu >= 64) + if (vcpu == VP_INVAL) { + local_irq_restore(flags); goto do_native; + } + + if (vcpu >= 64) + goto do_ex_hypercall; __set_bit(vcpu, (unsigned long *) &flush->processor_mask); @@ -123,7 +144,12 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus, status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, gva_n, 0, flush, NULL); } + goto check_status; + +do_ex_hypercall: + status = hyperv_flush_tlb_others_ex(cpus, info); +check_status: local_irq_restore(flags); if (!(status & HV_HYPERCALL_RESULT_MASK)) @@ -132,35 +158,22 @@ do_native: native_flush_tlb_others(cpus, info); } -static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, - const struct flush_tlb_info *info) +static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus, + const struct flush_tlb_info *info) { int nr_bank = 0, max_gvas, gva_n; struct hv_tlb_flush_ex **flush_pcpu; struct hv_tlb_flush_ex *flush; - u64 status = U64_MAX; - unsigned long flags; + u64 status; - trace_hyperv_mmu_flush_tlb_others(cpus, info); - - if (!hv_hypercall_pg) - goto do_native; - - if (cpumask_empty(cpus)) - return; - - local_irq_save(flags); + if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) + return U64_MAX; flush_pcpu = (struct hv_tlb_flush_ex **) this_cpu_ptr(hyperv_pcpu_input_arg); flush = *flush_pcpu; - if (unlikely(!flush)) { - local_irq_restore(flags); - goto do_native; - } - if (info->mm) { /* * AddressSpace argument must match the CR3 with PCID bits @@ -176,15 +189,10 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, flush->hv_vp_set.valid_bank_mask = 0; - if (!cpumask_equal(cpus, cpu_present_mask)) { - flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; - nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); - } - - if (!nr_bank) { - flush->hv_vp_set.format = HV_GENERIC_SET_ALL; - flush->flags |= HV_FLUSH_ALL_PROCESSORS; - } + flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; + nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus); + if (nr_bank < 0) + return U64_MAX; /* * We can flush not more than max_gvas with one hypercall. Flush the @@ -213,12 +221,7 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus, gva_n, nr_bank, flush, NULL); } - local_irq_restore(flags); - - if (!(status & HV_HYPERCALL_RESULT_MASK)) - return; -do_native: - native_flush_tlb_others(cpus, info); + return status; } void hyperv_setup_mmu_ops(void) @@ -226,11 +229,6 @@ void hyperv_setup_mmu_ops(void) if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED)) return; - if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) { - pr_info("Using hypercall for remote TLB flush\n"); - pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; - } else { - pr_info("Using ext hypercall for remote TLB flush\n"); - pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex; - } + pr_info("Using hypercall for remote TLB flush\n"); + pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; } diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 74a9e06b6cfd..130e81e10fc7 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -10,6 +10,7 @@ #include <asm/fixmap.h> #include <asm/mpspec.h> #include <asm/msr.h> +#include <asm/hardirq.h> #define ARCH_APICTIMER_STOPS_ON_C3 1 @@ -502,12 +503,19 @@ extern int default_check_phys_apicid_present(int phys_apicid); #endif /* CONFIG_X86_LOCAL_APIC */ +#ifdef CONFIG_SMP +bool apic_id_is_primary_thread(unsigned int id); +#else +static inline bool apic_id_is_primary_thread(unsigned int id) { return false; } +#endif + extern void irq_enter(void); extern void irq_exit(void); static inline void entering_irq(void) { irq_enter(); + kvm_set_cpu_l1tf_flush_l1d(); } static inline void entering_ack_irq(void) @@ -520,6 +528,7 @@ static inline void ipi_entering_ack_irq(void) { irq_enter(); ack_APIC_irq(); + kvm_set_cpu_l1tf_flush_l1d(); } static inline void exiting_irq(void) diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h index c356098b6fb9..4d4015ddcf26 100644 --- a/arch/x86/include/asm/apm.h +++ b/arch/x86/include/asm/apm.h @@ -7,8 +7,6 @@ #ifndef _ASM_X86_MACH_DEFAULT_APM_H #define _ASM_X86_MACH_DEFAULT_APM_H -#include <asm/nospec-branch.h> - #ifdef APM_ZERO_SEGS # define APM_DO_ZERO_SEGS \ "pushl %%ds\n\t" \ @@ -34,7 +32,6 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in, * N.B. We do NOT need a cld after the BIOS call * because we always save and restore the flags. */ - firmware_restrict_branch_speculation_start(); __asm__ __volatile__(APM_DO_ZERO_SEGS "pushl %%edi\n\t" "pushl %%ebp\n\t" @@ -47,7 +44,6 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in, "=S" (*esi) : "a" (func), "b" (ebx_in), "c" (ecx_in) : "memory", "cc"); - firmware_restrict_branch_speculation_end(); } static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in, @@ -60,7 +56,6 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in, * N.B. We do NOT need a cld after the BIOS call * because we always save and restore the flags. */ - firmware_restrict_branch_speculation_start(); __asm__ __volatile__(APM_DO_ZERO_SEGS "pushl %%edi\n\t" "pushl %%ebp\n\t" @@ -73,7 +68,6 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in, "=S" (si) : "a" (func), "b" (ebx_in), "c" (ecx_in) : "memory", "cc"); - firmware_restrict_branch_speculation_end(); return error; } diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 219faaec51df..990770f9e76b 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -46,6 +46,65 @@ #define _ASM_SI __ASM_REG(si) #define _ASM_DI __ASM_REG(di) +#ifndef __x86_64__ +/* 32 bit */ + +#define _ASM_ARG1 _ASM_AX +#define _ASM_ARG2 _ASM_DX +#define _ASM_ARG3 _ASM_CX + +#define _ASM_ARG1L eax +#define _ASM_ARG2L edx +#define _ASM_ARG3L ecx + +#define _ASM_ARG1W ax +#define _ASM_ARG2W dx +#define _ASM_ARG3W cx + +#define _ASM_ARG1B al +#define _ASM_ARG2B dl +#define _ASM_ARG3B cl + +#else +/* 64 bit */ + +#define _ASM_ARG1 _ASM_DI +#define _ASM_ARG2 _ASM_SI +#define _ASM_ARG3 _ASM_DX +#define _ASM_ARG4 _ASM_CX +#define _ASM_ARG5 r8 +#define _ASM_ARG6 r9 + +#define _ASM_ARG1Q rdi +#define _ASM_ARG2Q rsi +#define _ASM_ARG3Q rdx +#define _ASM_ARG4Q rcx +#define _ASM_ARG5Q r8 +#define _ASM_ARG6Q r9 + +#define _ASM_ARG1L edi +#define _ASM_ARG2L esi +#define _ASM_ARG3L edx +#define _ASM_ARG4L ecx +#define _ASM_ARG5L r8d +#define _ASM_ARG6L r9d + +#define _ASM_ARG1W di +#define _ASM_ARG2W si +#define _ASM_ARG3W dx +#define _ASM_ARG4W cx +#define _ASM_ARG5W r8w +#define _ASM_ARG6W r9w + +#define _ASM_ARG1B dil +#define _ASM_ARG2B sil +#define _ASM_ARG3B dl +#define _ASM_ARG4B cl +#define _ASM_ARG5B r8b +#define _ASM_ARG6B r9b + +#endif + /* * Macros to generate condition code outputs from inline assembly, * The output operand must be type "bool". diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 0db6bec95489..b143717b92b3 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -80,6 +80,7 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v) * true if the result is zero, or false for all * other cases. */ +#define arch_atomic_sub_and_test arch_atomic_sub_and_test static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) { GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e); @@ -91,6 +92,7 @@ static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) * * Atomically increments @v by 1. */ +#define arch_atomic_inc arch_atomic_inc static __always_inline void arch_atomic_inc(atomic_t *v) { asm volatile(LOCK_PREFIX "incl %0" @@ -103,6 +105,7 @@ static __always_inline void arch_atomic_inc(atomic_t *v) * * Atomically decrements @v by 1. */ +#define arch_atomic_dec arch_atomic_dec static __always_inline void arch_atomic_dec(atomic_t *v) { asm volatile(LOCK_PREFIX "decl %0" @@ -117,6 +120,7 @@ static __always_inline void arch_atomic_dec(atomic_t *v) * returns true if the result is 0, or false for all other * cases. */ +#define arch_atomic_dec_and_test arch_atomic_dec_and_test static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) { GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e); @@ -130,6 +134,7 @@ static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) * and returns true if the result is zero, or false for all * other cases. */ +#define arch_atomic_inc_and_test arch_atomic_inc_and_test static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) { GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e); @@ -144,6 +149,7 @@ static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ +#define arch_atomic_add_negative arch_atomic_add_negative static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v) { GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s); @@ -173,9 +179,6 @@ static __always_inline int arch_atomic_sub_return(int i, atomic_t *v) return arch_atomic_add_return(-i, v); } -#define arch_atomic_inc_return(v) (arch_atomic_add_return(1, v)) -#define arch_atomic_dec_return(v) (arch_atomic_sub_return(1, v)) - static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v) { return xadd(&v->counter, i); @@ -199,7 +202,7 @@ static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int n static inline int arch_atomic_xchg(atomic_t *v, int new) { - return xchg(&v->counter, new); + return arch_xchg(&v->counter, new); } static inline void arch_atomic_and(int i, atomic_t *v) @@ -253,27 +256,6 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) return val; } -/** - * __arch_atomic_add_unless - add unless the number is already a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as @v was not already @u. - * Returns the old value of @v. - */ -static __always_inline int __arch_atomic_add_unless(atomic_t *v, int a, int u) -{ - int c = arch_atomic_read(v); - - do { - if (unlikely(c == u)) - break; - } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); - - return c; -} - #ifdef CONFIG_X86_32 # include <asm/atomic64_32.h> #else diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h index 92212bf0484f..ef959f02d070 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -158,6 +158,7 @@ static inline long long arch_atomic64_inc_return(atomic64_t *v) "S" (v) : "memory", "ecx"); return a; } +#define arch_atomic64_inc_return arch_atomic64_inc_return static inline long long arch_atomic64_dec_return(atomic64_t *v) { @@ -166,6 +167,7 @@ static inline long long arch_atomic64_dec_return(atomic64_t *v) "S" (v) : "memory", "ecx"); return a; } +#define arch_atomic64_dec_return arch_atomic64_dec_return /** * arch_atomic64_add - add integer to atomic64 variable @@ -198,25 +200,12 @@ static inline long long arch_atomic64_sub(long long i, atomic64_t *v) } /** - * arch_atomic64_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer to type atomic64_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -static inline int arch_atomic64_sub_and_test(long long i, atomic64_t *v) -{ - return arch_atomic64_sub_return(i, v) == 0; -} - -/** * arch_atomic64_inc - increment atomic64 variable * @v: pointer to type atomic64_t * * Atomically increments @v by 1. */ +#define arch_atomic64_inc arch_atomic64_inc static inline void arch_atomic64_inc(atomic64_t *v) { __alternative_atomic64(inc, inc_return, /* no output */, @@ -229,6 +218,7 @@ static inline void arch_atomic64_inc(atomic64_t *v) * * Atomically decrements @v by 1. */ +#define arch_atomic64_dec arch_atomic64_dec static inline void arch_atomic64_dec(atomic64_t *v) { __alternative_atomic64(dec, dec_return, /* no output */, @@ -236,46 +226,6 @@ static inline void arch_atomic64_dec(atomic64_t *v) } /** - * arch_atomic64_dec_and_test - decrement and test - * @v: pointer to type atomic64_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -static inline int arch_atomic64_dec_and_test(atomic64_t *v) -{ - return arch_atomic64_dec_return(v) == 0; -} - -/** - * atomic64_inc_and_test - increment and test - * @v: pointer to type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -static inline int arch_atomic64_inc_and_test(atomic64_t *v) -{ - return arch_atomic64_inc_return(v) == 0; -} - -/** - * arch_atomic64_add_negative - add and test if negative - * @i: integer value to add - * @v: pointer to type atomic64_t - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -static inline int arch_atomic64_add_negative(long long i, atomic64_t *v) -{ - return arch_atomic64_add_return(i, v) < 0; -} - -/** * arch_atomic64_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t * @a: the amount to add to v... @@ -295,7 +245,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, long long a, return (int)a; } - +#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero static inline int arch_atomic64_inc_not_zero(atomic64_t *v) { int r; @@ -304,6 +254,7 @@ static inline int arch_atomic64_inc_not_zero(atomic64_t *v) return r; } +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive static inline long long arch_atomic64_dec_if_positive(atomic64_t *v) { long long r; diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h index 6106b59d3260..4343d9b4f30e 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -71,6 +71,7 @@ static inline void arch_atomic64_sub(long i, atomic64_t *v) * true if the result is zero, or false for all * other cases. */ +#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v) { GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e); @@ -82,6 +83,7 @@ static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v) * * Atomically increments @v by 1. */ +#define arch_atomic64_inc arch_atomic64_inc static __always_inline void arch_atomic64_inc(atomic64_t *v) { asm volatile(LOCK_PREFIX "incq %0" @@ -95,6 +97,7 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v) * * Atomically decrements @v by 1. */ +#define arch_atomic64_dec arch_atomic64_dec static __always_inline void arch_atomic64_dec(atomic64_t *v) { asm volatile(LOCK_PREFIX "decq %0" @@ -110,6 +113,7 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) * returns true if the result is 0, or false for all other * cases. */ +#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test static inline bool arch_atomic64_dec_and_test(atomic64_t *v) { GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e); @@ -123,6 +127,7 @@ static inline bool arch_atomic64_dec_and_test(atomic64_t *v) * and returns true if the result is zero, or false for all * other cases. */ +#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test static inline bool arch_atomic64_inc_and_test(atomic64_t *v) { GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e); @@ -137,6 +142,7 @@ static inline bool arch_atomic64_inc_and_test(atomic64_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ +#define arch_atomic64_add_negative arch_atomic64_add_negative static inline bool arch_atomic64_add_negative(long i, atomic64_t *v) { GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s); @@ -169,9 +175,6 @@ static inline long arch_atomic64_fetch_sub(long i, atomic64_t *v) return xadd(&v->counter, -i); } -#define arch_atomic64_inc_return(v) (arch_atomic64_add_return(1, (v))) -#define arch_atomic64_dec_return(v) (arch_atomic64_sub_return(1, (v))) - static inline long arch_atomic64_cmpxchg(atomic64_t *v, long old, long new) { return arch_cmpxchg(&v->counter, old, new); @@ -185,46 +188,7 @@ static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, l static inline long arch_atomic64_xchg(atomic64_t *v, long new) { - return xchg(&v->counter, new); -} - -/** - * arch_atomic64_add_unless - add unless the number is a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static inline bool arch_atomic64_add_unless(atomic64_t *v, long a, long u) -{ - s64 c = arch_atomic64_read(v); - do { - if (unlikely(c == u)) - return false; - } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); - return true; -} - -#define arch_atomic64_inc_not_zero(v) arch_atomic64_add_unless((v), 1, 0) - -/* - * arch_atomic64_dec_if_positive - decrement by 1 if old value positive - * @v: pointer of type atomic_t - * - * The function returns the old value of *v minus 1, even if - * the atomic variable, v, was not decremented. - */ -static inline long arch_atomic64_dec_if_positive(atomic64_t *v) -{ - s64 dec, c = arch_atomic64_read(v); - do { - dec = c - 1; - if (unlikely(dec < 0)) - break; - } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); - return dec; + return arch_xchg(&v->counter, new); } static inline void arch_atomic64_and(long i, atomic64_t *v) diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h index e3efd8a06066..a55d79b233d3 100644 --- a/arch/x86/include/asm/cmpxchg.h +++ b/arch/x86/include/asm/cmpxchg.h @@ -75,7 +75,7 @@ extern void __add_wrong_size(void) * use "asm volatile" and "memory" clobbers to prevent gcc from moving * information around. */ -#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "") +#define arch_xchg(ptr, v) __xchg_op((ptr), (v), xchg, "") /* * Atomic compare and exchange. Compare OLD with MEM, if identical, diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h index bfca3b346c74..072e5459fe2f 100644 --- a/arch/x86/include/asm/cmpxchg_64.h +++ b/arch/x86/include/asm/cmpxchg_64.h @@ -10,13 +10,13 @@ static inline void set_64bit(volatile u64 *ptr, u64 val) #define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ + arch_cmpxchg((ptr), (o), (n)); \ }) #define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 5701f5cecd31..89a048c2faec 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -219,6 +219,8 @@ #define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */ #define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */ #define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 (Zen) */ +#define X86_FEATURE_L1TF_PTEINV ( 7*32+29) /* "" L1TF workaround PTE inversion */ +#define X86_FEATURE_IBRS_ENHANCED ( 7*32+30) /* Enhanced IBRS */ /* Virtualization flags: Linux defined, word 8 */ #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ @@ -229,7 +231,7 @@ #define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer VMMCALL to VMCALL */ #define X86_FEATURE_XENPV ( 8*32+16) /* "" Xen paravirtual guest */ - +#define X86_FEATURE_EPT_AD ( 8*32+17) /* Intel Extended Page Table access-dirty bit */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ @@ -341,6 +343,7 @@ #define X86_FEATURE_PCONFIG (18*32+18) /* Intel PCONFIG */ #define X86_FEATURE_SPEC_CTRL (18*32+26) /* "" Speculation Control (IBRS + IBPB) */ #define X86_FEATURE_INTEL_STIBP (18*32+27) /* "" Single Thread Indirect Branch Predictors */ +#define X86_FEATURE_FLUSH_L1D (18*32+28) /* Flush L1D cache */ #define X86_FEATURE_ARCH_CAPABILITIES (18*32+29) /* IA32_ARCH_CAPABILITIES MSR (Intel) */ #define X86_FEATURE_SPEC_CTRL_SSBD (18*32+31) /* "" Speculative Store Bypass Disable */ @@ -373,5 +376,6 @@ #define X86_BUG_SPECTRE_V1 X86_BUG(15) /* CPU is affected by Spectre variant 1 attack with conditional branches */ #define X86_BUG_SPECTRE_V2 X86_BUG(16) /* CPU is affected by Spectre variant 2 attack with indirect branches */ #define X86_BUG_SPEC_STORE_BYPASS X86_BUG(17) /* CPU is affected by speculative store bypass attack */ +#define X86_BUG_L1TF X86_BUG(18) /* CPU is affected by L1 Terminal Fault */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/dmi.h b/arch/x86/include/asm/dmi.h index 0ab2ab27ad1f..b825cb201251 100644 --- a/arch/x86/include/asm/dmi.h +++ b/arch/x86/include/asm/dmi.h @@ -4,8 +4,8 @@ #include <linux/compiler.h> #include <linux/init.h> +#include <linux/io.h> -#include <asm/io.h> #include <asm/setup.h> static __always_inline __init void *dmi_alloc(unsigned len) diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h index 740a428acf1e..d9069bb26c7f 100644 --- a/arch/x86/include/asm/hardirq.h +++ b/arch/x86/include/asm/hardirq.h @@ -3,10 +3,12 @@ #define _ASM_X86_HARDIRQ_H #include <linux/threads.h> -#include <linux/irq.h> typedef struct { - unsigned int __softirq_pending; + u16 __softirq_pending; +#if IS_ENABLED(CONFIG_KVM_INTEL) + u8 kvm_cpu_l1tf_flush_l1d; +#endif unsigned int __nmi_count; /* arch dependent */ #ifdef CONFIG_X86_LOCAL_APIC unsigned int apic_timer_irqs; /* arch dependent */ @@ -58,4 +60,24 @@ extern u64 arch_irq_stat_cpu(unsigned int cpu); extern u64 arch_irq_stat(void); #define arch_irq_stat arch_irq_stat + +#if IS_ENABLED(CONFIG_KVM_INTEL) +static inline void kvm_set_cpu_l1tf_flush_l1d(void) +{ + __this_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 1); +} + +static inline void kvm_clear_cpu_l1tf_flush_l1d(void) +{ + __this_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 0); +} + +static inline bool kvm_get_cpu_l1tf_flush_l1d(void) +{ + return __this_cpu_read(irq_stat.kvm_cpu_l1tf_flush_l1d); +} +#else /* !IS_ENABLED(CONFIG_KVM_INTEL) */ +static inline void kvm_set_cpu_l1tf_flush_l1d(void) { } +#endif /* IS_ENABLED(CONFIG_KVM_INTEL) */ + #endif /* _ASM_X86_HARDIRQ_H */ diff --git a/arch/x86/include/asm/hw_breakpoint.h b/arch/x86/include/asm/hw_breakpoint.h index f59c39835a5a..a1f0e90d0818 100644 --- a/arch/x86/include/asm/hw_breakpoint.h +++ b/arch/x86/include/asm/hw_breakpoint.h @@ -49,11 +49,14 @@ static inline int hw_breakpoint_slots(int type) return HBP_NUM; } +struct perf_event_attr; struct perf_event; struct pmu; -extern int arch_check_bp_in_kernelspace(struct perf_event *bp); -extern int arch_validate_hwbkpt_settings(struct perf_event *bp); +extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +extern int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index cf090e584202..7ed08a7c3398 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -76,4 +76,17 @@ #define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ #define INTEL_FAM6_XEON_PHI_KNM 0x85 /* Knights Mill */ +/* Useful macros */ +#define INTEL_CPU_FAM_ANY(_family, _model, _driver_data) \ +{ \ + .vendor = X86_VENDOR_INTEL, \ + .family = _family, \ + .model = _model, \ + .feature = X86_FEATURE_ANY, \ + .driver_data = (kernel_ulong_t)&_driver_data \ +} + +#define INTEL_CPU_FAM6(_model, _driver_data) \ + INTEL_CPU_FAM_ANY(6, INTEL_FAM6_##_model, _driver_data) + #endif /* _ASM_X86_INTEL_FAMILY_H */ diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index fe04491130ae..52f815a80539 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -80,35 +80,6 @@ enum intel_mid_cpu_type { extern enum intel_mid_cpu_type __intel_mid_cpu_chip; -/** - * struct intel_mid_ops - Interface between intel-mid & sub archs - * @arch_setup: arch_setup function to re-initialize platform - * structures (x86_init, x86_platform_init) - * - * This structure can be extended if any new interface is required - * between intel-mid & its sub arch files. - */ -struct intel_mid_ops { - void (*arch_setup)(void); -}; - -/* Helper API's for INTEL_MID_OPS_INIT */ -#define DECLARE_INTEL_MID_OPS_INIT(cpuname, cpuid) \ - [cpuid] = get_##cpuname##_ops - -/* Maximum number of CPU ops */ -#define MAX_CPU_OPS(a) (sizeof(a)/sizeof(void *)) - -/* - * For every new cpu addition, a weak get_<cpuname>_ops() function needs be - * declared in arch/x86/platform/intel_mid/intel_mid_weak_decls.h. - */ -#define INTEL_MID_OPS_INIT { \ - DECLARE_INTEL_MID_OPS_INIT(penwell, INTEL_MID_CPU_CHIP_PENWELL), \ - DECLARE_INTEL_MID_OPS_INIT(cloverview, INTEL_MID_CPU_CHIP_CLOVERVIEW), \ - DECLARE_INTEL_MID_OPS_INIT(tangier, INTEL_MID_CPU_CHIP_TANGIER) \ -}; - #ifdef CONFIG_X86_INTEL_MID static inline enum intel_mid_cpu_type intel_mid_identify_cpu(void) @@ -136,20 +107,6 @@ enum intel_mid_timer_options { extern enum intel_mid_timer_options intel_mid_timer_options; -/* - * Penwell uses spread spectrum clock, so the freq number is not exactly - * the same as reported by MSR based on SDM. - */ -#define FSB_FREQ_83SKU 83200 -#define FSB_FREQ_100SKU 99840 -#define FSB_FREQ_133SKU 133000 - -#define FSB_FREQ_167SKU 167000 -#define FSB_FREQ_200SKU 200000 -#define FSB_FREQ_267SKU 267000 -#define FSB_FREQ_333SKU 333000 -#define FSB_FREQ_400SKU 400000 - /* Bus Select SoC Fuse value */ #define BSEL_SOC_FUSE_MASK 0x7 /* FSB 133MHz */ diff --git a/arch/x86/include/asm/intel_ds.h b/arch/x86/include/asm/intel_ds.h index 62a9f4966b42..ae26df1c2789 100644 --- a/arch/x86/include/asm/intel_ds.h +++ b/arch/x86/include/asm/intel_ds.h @@ -8,6 +8,7 @@ /* The maximal number of PEBS events: */ #define MAX_PEBS_EVENTS 8 +#define MAX_FIXED_PEBS_EVENTS 3 /* * A debug store configuration. @@ -23,7 +24,7 @@ struct debug_store { u64 pebs_index; u64 pebs_absolute_maximum; u64 pebs_interrupt_threshold; - u64 pebs_event_reset[MAX_PEBS_EVENTS]; + u64 pebs_event_reset[MAX_PEBS_EVENTS + MAX_FIXED_PEBS_EVENTS]; } __aligned(PAGE_SIZE); DECLARE_PER_CPU_PAGE_ALIGNED(struct debug_store, cpu_debug_store); diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 89f08955fff7..c14f2a74b2be 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -13,7 +13,9 @@ * Interrupt control: */ -static inline unsigned long native_save_fl(void) +/* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ +extern inline unsigned long native_save_fl(void); +extern inline unsigned long native_save_fl(void) { unsigned long flags; diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h index 367d99cff426..c8cec1b39b88 100644 --- a/arch/x86/include/asm/kprobes.h +++ b/arch/x86/include/asm/kprobes.h @@ -78,7 +78,7 @@ struct arch_specific_insn { * boostable = true: This instruction has been boosted: we have * added a relative jump after the instruction copy in insn, * so no single-step and fixup are needed (unless there's - * a post_handler or break_handler). + * a post_handler). */ bool boostable; bool if_modifier; @@ -111,9 +111,6 @@ struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_old_flags; unsigned long kprobe_saved_flags; - unsigned long *jprobe_saved_sp; - struct pt_regs jprobe_saved_regs; - kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/x86/include/asm/kvm_guest.h b/arch/x86/include/asm/kvm_guest.h deleted file mode 100644 index 46185263d9c2..000000000000 --- a/arch/x86/include/asm/kvm_guest.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_KVM_GUEST_H -#define _ASM_X86_KVM_GUEST_H - -int kvm_setup_vsyscall_timeinfo(void); - -#endif /* _ASM_X86_KVM_GUEST_H */ diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c13cd28d9d1b..acebb808c4b5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -17,6 +17,7 @@ #include <linux/tracepoint.h> #include <linux/cpumask.h> #include <linux/irq_work.h> +#include <linux/irq.h> #include <linux/kvm.h> #include <linux/kvm_para.h> @@ -713,6 +714,9 @@ struct kvm_vcpu_arch { /* be preempted when it's in kernel-mode(cpl=0) */ bool preempted_in_kernel; + + /* Flush the L1 Data cache for L1TF mitigation on VMENTER */ + bool l1tf_flush_l1d; }; struct kvm_lpage_info { @@ -881,6 +885,7 @@ struct kvm_vcpu_stat { u64 signal_exits; u64 irq_window_exits; u64 nmi_window_exits; + u64 l1d_flush; u64 halt_exits; u64 halt_successful_poll; u64 halt_attempted_poll; @@ -1413,6 +1418,7 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v); void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu); +u64 kvm_get_arch_capabilities(void); void kvm_define_shared_msr(unsigned index, u32 msr); int kvm_set_shared_msr(unsigned index, u64 val, u64 mask); diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 3aea2658323a..4c723632c036 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -7,7 +7,6 @@ #include <uapi/asm/kvm_para.h> extern void kvmclock_init(void); -extern int kvm_register_clock(char *txt); #ifdef CONFIG_KVM_GUEST bool kvm_check_and_clear_guest_paused(void); diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index bbc796eb0a3b..eeeb9289c764 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -71,12 +71,7 @@ struct ldt_struct { static inline void *ldt_slot_va(int slot) { -#ifdef CONFIG_X86_64 return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot); -#else - BUG(); - return (void *)fix_to_virt(FIX_HOLE); -#endif } /* diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 3cd14311edfa..19886fef1dfc 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -9,6 +9,8 @@ #include <asm/hyperv-tlfs.h> #include <asm/nospec-branch.h> +#define VP_INVAL U32_MAX + struct ms_hyperv_info { u32 features; u32 misc_features; @@ -20,7 +22,6 @@ struct ms_hyperv_info { extern struct ms_hyperv_info ms_hyperv; - /* * Generate the guest ID. */ @@ -193,6 +194,40 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) return hv_status; } +/* Fast hypercall with 16 bytes of input */ +static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2) +{ + u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; + +#ifdef CONFIG_X86_64 + { + __asm__ __volatile__("mov %4, %%r8\n" + CALL_NOSPEC + : "=a" (hv_status), ASM_CALL_CONSTRAINT, + "+c" (control), "+d" (input1) + : "r" (input2), + THUNK_TARGET(hv_hypercall_pg) + : "cc", "r8", "r9", "r10", "r11"); + } +#else + { + u32 input1_hi = upper_32_bits(input1); + u32 input1_lo = lower_32_bits(input1); + u32 input2_hi = upper_32_bits(input2); + u32 input2_lo = lower_32_bits(input2); + + __asm__ __volatile__ (CALL_NOSPEC + : "=A"(hv_status), + "+c"(input1_lo), ASM_CALL_CONSTRAINT + : "A" (control), "b" (input1_hi), + "D"(input2_hi), "S"(input2_lo), + THUNK_TARGET(hv_hypercall_pg) + : "cc"); + } +#endif + return hv_status; +} + /* * Rep hypercalls. Callers of this functions are supposed to ensure that * rep_count and varhead_size comply with Hyper-V hypercall definition. @@ -281,6 +316,8 @@ static inline int cpumask_to_vpset(struct hv_vpset *vpset, */ for_each_cpu(cpu, cpus) { vcpu = hv_cpu_number_to_vp_number(cpu); + if (vcpu == VP_INVAL) + return -1; vcpu_bank = vcpu / 64; vcpu_offset = vcpu % 64; __set_bit(vcpu_offset, (unsigned long *) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 68b2c3150de1..4731f0cf97c5 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -70,12 +70,19 @@ #define MSR_IA32_ARCH_CAPABILITIES 0x0000010a #define ARCH_CAP_RDCL_NO (1 << 0) /* Not susceptible to Meltdown */ #define ARCH_CAP_IBRS_ALL (1 << 1) /* Enhanced IBRS support */ +#define ARCH_CAP_SKIP_VMENTRY_L1DFLUSH (1 << 3) /* Skip L1D flush on vmentry */ #define ARCH_CAP_SSB_NO (1 << 4) /* * Not susceptible to Speculative Store Bypass * attack, so no Speculative Store Bypass * control required. */ +#define MSR_IA32_FLUSH_CMD 0x0000010b +#define L1D_FLUSH (1 << 0) /* + * Writeback and invalidate the + * L1 data cache. + */ + #define MSR_IA32_BBL_CR_CTL 0x00000119 #define MSR_IA32_BBL_CR_CTL3 0x0000011e diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index f6f6c63da62f..fd2a8c1b88bc 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -214,7 +214,7 @@ enum spectre_v2_mitigation { SPECTRE_V2_RETPOLINE_MINIMAL_AMD, SPECTRE_V2_RETPOLINE_GENERIC, SPECTRE_V2_RETPOLINE_AMD, - SPECTRE_V2_IBRS, + SPECTRE_V2_IBRS_ENHANCED, }; /* The Speculative Store Bypass disable variants */ diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index 9c9dc579bd7d..46f516dd80ce 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -88,6 +88,7 @@ struct orc_entry { unsigned sp_reg:4; unsigned bp_reg:4; unsigned type:2; + unsigned end:1; } __packed; /* @@ -101,6 +102,7 @@ struct unwind_hint { s16 sp_offset; u8 sp_reg; u8 type; + u8 end; }; #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h index aa30c3241ea7..0d5c739eebd7 100644 --- a/arch/x86/include/asm/page_32_types.h +++ b/arch/x86/include/asm/page_32_types.h @@ -29,8 +29,13 @@ #define N_EXCEPTION_STACKS 1 #ifdef CONFIG_X86_PAE -/* 44=32+12, the limit we can fit into an unsigned long pfn */ -#define __PHYSICAL_MASK_SHIFT 44 +/* + * This is beyond the 44 bit limit imposed by the 32bit long pfns, + * but we need the full mask to make sure inverted PROT_NONE + * entries have all the host bits set in a guest. + * The real limit is still 44 bits. + */ +#define __PHYSICAL_MASK_SHIFT 52 #define __VIRTUAL_MASK_SHIFT 32 #else /* !CONFIG_X86_PAE */ diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index a06b07399d17..e9202a0de8f0 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -450,9 +450,10 @@ do { \ bool __ret; \ typeof(pcp1) __o1 = (o1), __n1 = (n1); \ typeof(pcp2) __o2 = (o2), __n2 = (n2); \ - asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t" \ - : "=a" (__ret), "+m" (pcp1), "+m" (pcp2), "+d" (__o2) \ - : "b" (__n1), "c" (__n2), "a" (__o1)); \ + asm volatile("cmpxchg8b "__percpu_arg(1) \ + CC_SET(z) \ + : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \ + : "b" (__n1), "c" (__n2)); \ __ret; \ }) diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h index 685ffe8a0eaf..24c6cf5f16b7 100644 --- a/arch/x86/include/asm/pgtable-2level.h +++ b/arch/x86/include/asm/pgtable-2level.h @@ -19,6 +19,9 @@ static inline void native_set_pte(pte_t *ptep , pte_t pte) static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { +#ifdef CONFIG_PAGE_TABLE_ISOLATION + pmd.pud.p4d.pgd = pti_set_user_pgtbl(&pmdp->pud.p4d.pgd, pmd.pud.p4d.pgd); +#endif *pmdp = pmd; } @@ -58,6 +61,9 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) #ifdef CONFIG_SMP static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) { +#ifdef CONFIG_PAGE_TABLE_ISOLATION + pti_set_user_pgtbl(&xp->pud.p4d.pgd, __pgd(0)); +#endif return __pmd(xchg((pmdval_t *)xp, 0)); } #else @@ -67,6 +73,9 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) #ifdef CONFIG_SMP static inline pud_t native_pudp_get_and_clear(pud_t *xp) { +#ifdef CONFIG_PAGE_TABLE_ISOLATION + pti_set_user_pgtbl(&xp->p4d.pgd, __pgd(0)); +#endif return __pud(xchg((pudval_t *)xp, 0)); } #else @@ -95,4 +104,21 @@ static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshi #define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) +/* No inverted PFNs on 2 level page tables */ + +static inline u64 protnone_mask(u64 val) +{ + return 0; +} + +static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask) +{ + return val; +} + +static inline bool __pte_needs_invert(u64 val) +{ + return false; +} + #endif /* _ASM_X86_PGTABLE_2LEVEL_H */ diff --git a/arch/x86/include/asm/pgtable-2level_types.h b/arch/x86/include/asm/pgtable-2level_types.h index f982ef808e7e..6deb6cd236e3 100644 --- a/arch/x86/include/asm/pgtable-2level_types.h +++ b/arch/x86/include/asm/pgtable-2level_types.h @@ -35,4 +35,7 @@ typedef union { #define PTRS_PER_PTE 1024 +/* This covers all VMSPLIT_* and VMSPLIT_*_OPT variants */ +#define PGD_KERNEL_START (CONFIG_PAGE_OFFSET >> PGDIR_SHIFT) + #endif /* _ASM_X86_PGTABLE_2LEVEL_DEFS_H */ diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h index f24df59c40b2..a564084c6141 100644 --- a/arch/x86/include/asm/pgtable-3level.h +++ b/arch/x86/include/asm/pgtable-3level.h @@ -98,6 +98,9 @@ static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) static inline void native_set_pud(pud_t *pudp, pud_t pud) { +#ifdef CONFIG_PAGE_TABLE_ISOLATION + pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); +#endif set_64bit((unsigned long long *)(pudp), native_pud_val(pud)); } @@ -229,6 +232,10 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp) { union split_pud res, *orig = (union split_pud *)pudp; +#ifdef CONFIG_PAGE_TABLE_ISOLATION + pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0)); +#endif + /* xchg acts as a barrier before setting of the high bits */ res.pud_low = xchg(&orig->pud_low, 0); res.pud_high = orig->pud_high; @@ -241,12 +248,43 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp) #endif /* Encode and de-code a swap entry */ +#define SWP_TYPE_BITS 5 + +#define SWP_OFFSET_FIRST_BIT (_PAGE_BIT_PROTNONE + 1) + +/* We always extract/encode the offset by shifting it all the way up, and then down again */ +#define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS) + #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5) #define __swp_type(x) (((x).val) & 0x1f) #define __swp_offset(x) ((x).val >> 5) #define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5}) -#define __pte_to_swp_entry(pte) ((swp_entry_t){ (pte).pte_high }) -#define __swp_entry_to_pte(x) ((pte_t){ { .pte_high = (x).val } }) + +/* + * Normally, __swp_entry() converts from arch-independent swp_entry_t to + * arch-dependent swp_entry_t, and __swp_entry_to_pte() just stores the result + * to pte. But here we have 32bit swp_entry_t and 64bit pte, and need to use the + * whole 64 bits. Thus, we shift the "real" arch-dependent conversion to + * __swp_entry_to_pte() through the following helper macro based on 64bit + * __swp_entry(). + */ +#define __swp_pteval_entry(type, offset) ((pteval_t) { \ + (~(pteval_t)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \ + | ((pteval_t)(type) << (64 - SWP_TYPE_BITS)) }) + +#define __swp_entry_to_pte(x) ((pte_t){ .pte = \ + __swp_pteval_entry(__swp_type(x), __swp_offset(x)) }) +/* + * Analogically, __pte_to_swp_entry() doesn't just extract the arch-dependent + * swp_entry_t, but also has to convert it from 64bit to the 32bit + * intermediate representation, using the following macros based on 64bit + * __swp_type() and __swp_offset(). + */ +#define __pteval_swp_type(x) ((unsigned long)((x).pte >> (64 - SWP_TYPE_BITS))) +#define __pteval_swp_offset(x) ((unsigned long)(~((x).pte) << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT)) + +#define __pte_to_swp_entry(pte) (__swp_entry(__pteval_swp_type(pte), \ + __pteval_swp_offset(pte))) #define gup_get_pte gup_get_pte /* @@ -295,4 +333,6 @@ static inline pte_t gup_get_pte(pte_t *ptep) return pte; } +#include <asm/pgtable-invert.h> + #endif /* _ASM_X86_PGTABLE_3LEVEL_H */ diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h index 6a59a6d0cc50..858358a82b14 100644 --- a/arch/x86/include/asm/pgtable-3level_types.h +++ b/arch/x86/include/asm/pgtable-3level_types.h @@ -21,9 +21,10 @@ typedef union { #endif /* !__ASSEMBLY__ */ #ifdef CONFIG_PARAVIRT -#define SHARED_KERNEL_PMD (pv_info.shared_kernel_pmd) +#define SHARED_KERNEL_PMD ((!static_cpu_has(X86_FEATURE_PTI) && \ + (pv_info.shared_kernel_pmd))) #else -#define SHARED_KERNEL_PMD 1 +#define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI)) #endif /* @@ -45,5 +46,6 @@ typedef union { #define PTRS_PER_PTE 512 #define MAX_POSSIBLE_PHYSMEM_BITS 36 +#define PGD_KERNEL_START (CONFIG_PAGE_OFFSET >> PGDIR_SHIFT) #endif /* _ASM_X86_PGTABLE_3LEVEL_DEFS_H */ diff --git a/arch/x86/include/asm/pgtable-invert.h b/arch/x86/include/asm/pgtable-invert.h new file mode 100644 index 000000000000..44b1203ece12 --- /dev/null +++ b/arch/x86/include/asm/pgtable-invert.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PGTABLE_INVERT_H +#define _ASM_PGTABLE_INVERT_H 1 + +#ifndef __ASSEMBLY__ + +static inline bool __pte_needs_invert(u64 val) +{ + return !(val & _PAGE_PRESENT); +} + +/* Get a mask to xor with the page table entry to get the correct pfn. */ +static inline u64 protnone_mask(u64 val) +{ + return __pte_needs_invert(val) ? ~0ull : 0; +} + +static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask) +{ + /* + * When a PTE transitions from NONE to !NONE or vice-versa + * invert the PFN part to stop speculation. + * pte_pfn undoes this when needed. + */ + if (__pte_needs_invert(oldval) != __pte_needs_invert(val)) + val = (val & ~mask) | (~val & mask); + return val; +} + +#endif /* __ASSEMBLY__ */ + +#endif diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 5715647fc4fe..e4ffa565a69f 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -30,11 +30,14 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd); void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd); void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user); void ptdump_walk_pgd_level_checkwx(void); +void ptdump_walk_user_pgd_level_checkwx(void); #ifdef CONFIG_DEBUG_WX -#define debug_checkwx() ptdump_walk_pgd_level_checkwx() +#define debug_checkwx() ptdump_walk_pgd_level_checkwx() +#define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx() #else -#define debug_checkwx() do { } while (0) +#define debug_checkwx() do { } while (0) +#define debug_checkwx_user() do { } while (0) #endif /* @@ -185,19 +188,29 @@ static inline int pte_special(pte_t pte) return pte_flags(pte) & _PAGE_SPECIAL; } +/* Entries that were set to PROT_NONE are inverted */ + +static inline u64 protnone_mask(u64 val); + static inline unsigned long pte_pfn(pte_t pte) { - return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT; + phys_addr_t pfn = pte_val(pte); + pfn ^= protnone_mask(pfn); + return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT; } static inline unsigned long pmd_pfn(pmd_t pmd) { - return (pmd_val(pmd) & pmd_pfn_mask(pmd)) >> PAGE_SHIFT; + phys_addr_t pfn = pmd_val(pmd); + pfn ^= protnone_mask(pfn); + return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT; } static inline unsigned long pud_pfn(pud_t pud) { - return (pud_val(pud) & pud_pfn_mask(pud)) >> PAGE_SHIFT; + phys_addr_t pfn = pud_val(pud); + pfn ^= protnone_mask(pfn); + return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT; } static inline unsigned long p4d_pfn(p4d_t p4d) @@ -400,11 +413,6 @@ static inline pmd_t pmd_mkwrite(pmd_t pmd) return pmd_set_flags(pmd, _PAGE_RW); } -static inline pmd_t pmd_mknotpresent(pmd_t pmd) -{ - return pmd_clear_flags(pmd, _PAGE_PRESENT | _PAGE_PROTNONE); -} - static inline pud_t pud_set_flags(pud_t pud, pudval_t set) { pudval_t v = native_pud_val(pud); @@ -459,11 +467,6 @@ static inline pud_t pud_mkwrite(pud_t pud) return pud_set_flags(pud, _PAGE_RW); } -static inline pud_t pud_mknotpresent(pud_t pud) -{ - return pud_clear_flags(pud, _PAGE_PRESENT | _PAGE_PROTNONE); -} - #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline int pte_soft_dirty(pte_t pte) { @@ -545,25 +548,45 @@ static inline pgprotval_t check_pgprot(pgprot_t pgprot) static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) { - return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) | - check_pgprot(pgprot)); + phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT; + pfn ^= protnone_mask(pgprot_val(pgprot)); + pfn &= PTE_PFN_MASK; + return __pte(pfn | check_pgprot(pgprot)); } static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) { - return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) | - check_pgprot(pgprot)); + phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT; + pfn ^= protnone_mask(pgprot_val(pgprot)); + pfn &= PHYSICAL_PMD_PAGE_MASK; + return __pmd(pfn | check_pgprot(pgprot)); } static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot) { - return __pud(((phys_addr_t)page_nr << PAGE_SHIFT) | - check_pgprot(pgprot)); + phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT; + pfn ^= protnone_mask(pgprot_val(pgprot)); + pfn &= PHYSICAL_PUD_PAGE_MASK; + return __pud(pfn | check_pgprot(pgprot)); +} + +static inline pmd_t pmd_mknotpresent(pmd_t pmd) +{ + return pfn_pmd(pmd_pfn(pmd), + __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); +} + +static inline pud_t pud_mknotpresent(pud_t pud) +{ + return pfn_pud(pud_pfn(pud), + __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); } +static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); + static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - pteval_t val = pte_val(pte); + pteval_t val = pte_val(pte), oldval = val; /* * Chop off the NX bit (if present), and add the NX portion of @@ -571,17 +594,17 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) */ val &= _PAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK; - + val = flip_protnone_guard(oldval, val, PTE_PFN_MASK); return __pte(val); } static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - pmdval_t val = pmd_val(pmd); + pmdval_t val = pmd_val(pmd), oldval = val; val &= _HPAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; - + val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); return __pmd(val); } @@ -640,8 +663,31 @@ static inline int is_new_memtype_allowed(u64 paddr, unsigned long size, pmd_t *populate_extra_pmd(unsigned long vaddr); pte_t *populate_extra_pte(unsigned long vaddr); + +#ifdef CONFIG_PAGE_TABLE_ISOLATION +pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd); + +/* + * Take a PGD location (pgdp) and a pgd value that needs to be set there. + * Populates the user and returns the resulting PGD that must be set in + * the kernel copy of the page tables. + */ +static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) +{ + if (!static_cpu_has(X86_FEATURE_PTI)) + return pgd; + return __pti_set_user_pgtbl(pgdp, pgd); +} +#else /* CONFIG_PAGE_TABLE_ISOLATION */ +static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) +{ + return pgd; +} +#endif /* CONFIG_PAGE_TABLE_ISOLATION */ + #endif /* __ASSEMBLY__ */ + #ifdef CONFIG_X86_32 # include <asm/pgtable_32.h> #else @@ -1154,6 +1200,70 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, } } #endif +/* + * Page table pages are page-aligned. The lower half of the top + * level is used for userspace and the top half for the kernel. + * + * Returns true for parts of the PGD that map userspace and + * false for the parts that map the kernel. + */ +static inline bool pgdp_maps_userspace(void *__ptr) +{ + unsigned long ptr = (unsigned long)__ptr; + + return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START); +} + +static inline int pgd_large(pgd_t pgd) { return 0; } + +#ifdef CONFIG_PAGE_TABLE_ISOLATION +/* + * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages + * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and + * the user one is in the last 4k. To switch between them, you + * just need to flip the 12th bit in their addresses. + */ +#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT + +/* + * This generates better code than the inline assembly in + * __set_bit(). + */ +static inline void *ptr_set_bit(void *ptr, int bit) +{ + unsigned long __ptr = (unsigned long)ptr; + + __ptr |= BIT(bit); + return (void *)__ptr; +} +static inline void *ptr_clear_bit(void *ptr, int bit) +{ + unsigned long __ptr = (unsigned long)ptr; + + __ptr &= ~BIT(bit); + return (void *)__ptr; +} + +static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) +{ + return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); +} + +static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) +{ + return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); +} + +static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) +{ + return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); +} + +static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) +{ + return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); +} +#endif /* CONFIG_PAGE_TABLE_ISOLATION */ /* * clone_pgd_range(pgd_t *dst, pgd_t *src, int count); @@ -1320,6 +1430,14 @@ static inline bool pud_access_permitted(pud_t pud, bool write) return __pte_access_permitted(pud_val(pud), write); } +#define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1 +extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot); + +static inline bool arch_has_pfn_modify_check(void) +{ + return boot_cpu_has_bug(X86_BUG_L1TF); +} + #include <asm-generic/pgtable.h> #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index 88a056b01db4..b3ec519e3982 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -34,8 +34,6 @@ static inline void check_pgt_cache(void) { } void paging_init(void); void sync_initial_page_table(void); -static inline int pgd_large(pgd_t pgd) { return 0; } - /* * Define this if things work differently on an i386 and an i486: * it will (on an i486) warn about kernel memory accesses that are diff --git a/arch/x86/include/asm/pgtable_32_types.h b/arch/x86/include/asm/pgtable_32_types.h index d9a001a4a872..b0bc0fff5f1f 100644 --- a/arch/x86/include/asm/pgtable_32_types.h +++ b/arch/x86/include/asm/pgtable_32_types.h @@ -50,13 +50,18 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */ ((FIXADDR_TOT_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1)) \ & PMD_MASK) -#define PKMAP_BASE \ +#define LDT_BASE_ADDR \ ((CPU_ENTRY_AREA_BASE - PAGE_SIZE) & PMD_MASK) +#define LDT_END_ADDR (LDT_BASE_ADDR + PMD_SIZE) + +#define PKMAP_BASE \ + ((LDT_BASE_ADDR - PAGE_SIZE) & PMD_MASK) + #ifdef CONFIG_HIGHMEM # define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE) #else -# define VMALLOC_END (CPU_ENTRY_AREA_BASE - 2 * PAGE_SIZE) +# define VMALLOC_END (LDT_BASE_ADDR - 2 * PAGE_SIZE) #endif #define MODULES_VADDR VMALLOC_START diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 3c5385f9a88f..f773d5e6c8cc 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -132,90 +132,6 @@ static inline pud_t native_pudp_get_and_clear(pud_t *xp) #endif } -#ifdef CONFIG_PAGE_TABLE_ISOLATION -/* - * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages - * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and - * the user one is in the last 4k. To switch between them, you - * just need to flip the 12th bit in their addresses. - */ -#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT - -/* - * This generates better code than the inline assembly in - * __set_bit(). - */ -static inline void *ptr_set_bit(void *ptr, int bit) -{ - unsigned long __ptr = (unsigned long)ptr; - - __ptr |= BIT(bit); - return (void *)__ptr; -} -static inline void *ptr_clear_bit(void *ptr, int bit) -{ - unsigned long __ptr = (unsigned long)ptr; - - __ptr &= ~BIT(bit); - return (void *)__ptr; -} - -static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) -{ - return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); -} - -static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) -{ - return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); -} - -static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) -{ - return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); -} - -static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) -{ - return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); -} -#endif /* CONFIG_PAGE_TABLE_ISOLATION */ - -/* - * Page table pages are page-aligned. The lower half of the top - * level is used for userspace and the top half for the kernel. - * - * Returns true for parts of the PGD that map userspace and - * false for the parts that map the kernel. - */ -static inline bool pgdp_maps_userspace(void *__ptr) -{ - unsigned long ptr = (unsigned long)__ptr; - - return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2); -} - -#ifdef CONFIG_PAGE_TABLE_ISOLATION -pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd); - -/* - * Take a PGD location (pgdp) and a pgd value that needs to be set there. - * Populates the user and returns the resulting PGD that must be set in - * the kernel copy of the page tables. - */ -static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) -{ - if (!static_cpu_has(X86_FEATURE_PTI)) - return pgd; - return __pti_set_user_pgd(pgdp, pgd); -} -#else -static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) -{ - return pgd; -} -#endif - static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) { pgd_t pgd; @@ -226,7 +142,7 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) } pgd = native_make_pgd(native_p4d_val(p4d)); - pgd = pti_set_user_pgd((pgd_t *)p4dp, pgd); + pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); *p4dp = native_make_p4d(native_pgd_val(pgd)); } @@ -237,7 +153,7 @@ static inline void native_p4d_clear(p4d_t *p4d) static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) { - *pgdp = pti_set_user_pgd(pgdp, pgd); + *pgdp = pti_set_user_pgtbl(pgdp, pgd); } static inline void native_pgd_clear(pgd_t *pgd) @@ -255,7 +171,6 @@ extern void sync_global_pgds(unsigned long start, unsigned long end); /* * Level 4 access. */ -static inline int pgd_large(pgd_t pgd) { return 0; } #define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE) /* PUD - Level3 access */ @@ -273,7 +188,7 @@ static inline int pgd_large(pgd_t pgd) { return 0; } * * | ... | 11| 10| 9|8|7|6|5| 4| 3|2| 1|0| <- bit number * | ... |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names - * | OFFSET (14->63) | TYPE (9-13) |0|0|X|X| X| X|X|SD|0| <- swp entry + * | TYPE (59-63) | ~OFFSET (9-58) |0|0|X|X| X| X|X|SD|0| <- swp entry * * G (8) is aliased and used as a PROT_NONE indicator for * !present ptes. We need to start storing swap entries above @@ -286,20 +201,34 @@ static inline int pgd_large(pgd_t pgd) { return 0; } * * Bit 7 in swp entry should be 0 because pmd_present checks not only P, * but also L and G. + * + * The offset is inverted by a binary not operation to make the high + * physical bits set. */ -#define SWP_TYPE_FIRST_BIT (_PAGE_BIT_PROTNONE + 1) -#define SWP_TYPE_BITS 5 -/* Place the offset above the type: */ -#define SWP_OFFSET_FIRST_BIT (SWP_TYPE_FIRST_BIT + SWP_TYPE_BITS) +#define SWP_TYPE_BITS 5 + +#define SWP_OFFSET_FIRST_BIT (_PAGE_BIT_PROTNONE + 1) + +/* We always extract/encode the offset by shifting it all the way up, and then down again */ +#define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT+SWP_TYPE_BITS) #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) -#define __swp_type(x) (((x).val >> (SWP_TYPE_FIRST_BIT)) \ - & ((1U << SWP_TYPE_BITS) - 1)) -#define __swp_offset(x) ((x).val >> SWP_OFFSET_FIRST_BIT) -#define __swp_entry(type, offset) ((swp_entry_t) { \ - ((type) << (SWP_TYPE_FIRST_BIT)) \ - | ((offset) << SWP_OFFSET_FIRST_BIT) }) +/* Extract the high bits for type */ +#define __swp_type(x) ((x).val >> (64 - SWP_TYPE_BITS)) + +/* Shift up (to get rid of type), then down to get value */ +#define __swp_offset(x) (~(x).val << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT) + +/* + * Shift the offset up "too far" by TYPE bits, then down again + * The offset is inverted by a binary not operation to make the high + * physical bits set. + */ +#define __swp_entry(type, offset) ((swp_entry_t) { \ + (~(unsigned long)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \ + | ((unsigned long)(type) << (64-SWP_TYPE_BITS)) }) + #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val((pmd)) }) #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) @@ -343,5 +272,7 @@ static inline bool gup_fast_permitted(unsigned long start, int nr_pages, return true; } +#include <asm/pgtable-invert.h> + #endif /* !__ASSEMBLY__ */ #endif /* _ASM_X86_PGTABLE_64_H */ diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index 054765ab2da2..04edd2d58211 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -115,6 +115,7 @@ extern unsigned int ptrs_per_p4d; #define LDT_PGD_ENTRY_L5 -112UL #define LDT_PGD_ENTRY (pgtable_l5_enabled() ? LDT_PGD_ENTRY_L5 : LDT_PGD_ENTRY_L4) #define LDT_BASE_ADDR (LDT_PGD_ENTRY << PGDIR_SHIFT) +#define LDT_END_ADDR (LDT_BASE_ADDR + PGDIR_SIZE) #define __VMALLOC_BASE_L4 0xffffc90000000000UL #define __VMALLOC_BASE_L5 0xffa0000000000000UL @@ -153,4 +154,6 @@ extern unsigned int ptrs_per_p4d; #define EARLY_DYNAMIC_PAGE_TABLES 64 +#define PGD_KERNEL_START ((PAGE_SIZE / 2) / sizeof(pgd_t)) + #endif /* _ASM_X86_PGTABLE_64_DEFS_H */ diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 99fff853c944..b64acb08a62b 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -50,6 +50,7 @@ #define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL) #define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1) #define _PAGE_SOFTW2 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2) +#define _PAGE_SOFTW3 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW3) #define _PAGE_PAT (_AT(pteval_t, 1) << _PAGE_BIT_PAT) #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE) #define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL) @@ -266,14 +267,37 @@ typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; typedef struct { pgdval_t pgd; } pgd_t; +#ifdef CONFIG_X86_PAE + +/* + * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't + * use it here. + */ + +#define PGD_PAE_PAGE_MASK ((signed long)PAGE_MASK) +#define PGD_PAE_PHYS_MASK (((1ULL << __PHYSICAL_MASK_SHIFT)-1) & PGD_PAE_PAGE_MASK) + +/* + * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries. + * All other bits are Reserved MBZ + */ +#define PGD_ALLOWED_BITS (PGD_PAE_PHYS_MASK | _PAGE_PRESENT | \ + _PAGE_PWT | _PAGE_PCD | \ + _PAGE_SOFTW1 | _PAGE_SOFTW2 | _PAGE_SOFTW3) + +#else +/* No need to mask any bits for !PAE */ +#define PGD_ALLOWED_BITS (~0ULL) +#endif + static inline pgd_t native_make_pgd(pgdval_t val) { - return (pgd_t) { val }; + return (pgd_t) { val & PGD_ALLOWED_BITS }; } static inline pgdval_t native_pgd_val(pgd_t pgd) { - return pgd.pgd; + return pgd.pgd & PGD_ALLOWED_BITS; } static inline pgdval_t pgd_flags(pgd_t pgd) diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h index 625a52a5594f..02c2cbda4a74 100644 --- a/arch/x86/include/asm/processor-flags.h +++ b/arch/x86/include/asm/processor-flags.h @@ -39,10 +39,6 @@ #define CR3_PCID_MASK 0xFFFull #define CR3_NOFLUSH BIT_ULL(63) -#ifdef CONFIG_PAGE_TABLE_ISOLATION -# define X86_CR3_PTI_PCID_USER_BIT 11 -#endif - #else /* * CR3_ADDR_MASK needs at least bits 31:5 set on PAE systems, and we save @@ -53,4 +49,8 @@ #define CR3_NOFLUSH 0 #endif +#ifdef CONFIG_PAGE_TABLE_ISOLATION +# define X86_CR3_PTI_PCID_USER_BIT 11 +#endif + #endif /* _ASM_X86_PROCESSOR_FLAGS_H */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index cfd29ee8c3da..682286aca881 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -181,6 +181,11 @@ extern const struct seq_operations cpuinfo_op; extern void cpu_detect(struct cpuinfo_x86 *c); +static inline unsigned long l1tf_pfn_limit(void) +{ + return BIT(boot_cpu_data.x86_phys_bits - 1 - PAGE_SHIFT) - 1; +} + extern void early_cpu_init(void); extern void identify_boot_cpu(void); extern void identify_secondary_cpu(struct cpuinfo_x86 *); @@ -966,6 +971,7 @@ static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) extern unsigned long arch_align_stack(unsigned long sp); extern void free_init_pages(char *what, unsigned long begin, unsigned long end); +extern void free_kernel_image_pages(void *begin, void *end); void default_idle(void); #ifdef CONFIG_XEN @@ -977,4 +983,16 @@ bool xen_set_default_idle(void); void stop_this_cpu(void *dummy); void df_debug(struct pt_regs *regs, long error_code); void microcode_check(void); + +enum l1tf_mitigations { + L1TF_MITIGATION_OFF, + L1TF_MITIGATION_FLUSH_NOWARN, + L1TF_MITIGATION_FLUSH, + L1TF_MITIGATION_FLUSH_NOSMT, + L1TF_MITIGATION_FULL, + L1TF_MITIGATION_FULL_FORCE +}; + +extern enum l1tf_mitigations l1tf_mitigation; + #endif /* _ASM_X86_PROCESSOR_H */ diff --git a/arch/x86/include/asm/pti.h b/arch/x86/include/asm/pti.h index 38a17f1d5c9d..5df09a0b80b8 100644 --- a/arch/x86/include/asm/pti.h +++ b/arch/x86/include/asm/pti.h @@ -6,10 +6,9 @@ #ifdef CONFIG_PAGE_TABLE_ISOLATION extern void pti_init(void); extern void pti_check_boottime_disable(void); -extern void pti_clone_kernel_text(void); +extern void pti_finalize(void); #else static inline void pti_check_boottime_disable(void) { } -static inline void pti_clone_kernel_text(void) { } #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h index 9ef5ee03d2d7..159622ee0674 100644 --- a/arch/x86/include/asm/qspinlock_paravirt.h +++ b/arch/x86/include/asm/qspinlock_paravirt.h @@ -43,7 +43,7 @@ asm (".pushsection .text;" "push %rdx;" "mov $0x1,%eax;" "xor %edx,%edx;" - "lock cmpxchg %dl,(%rdi);" + LOCK_PREFIX "cmpxchg %dl,(%rdi);" "cmp $0x1,%al;" "jne .slowpath;" "pop %rdx;" diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h index 4cf11d88d3b3..19b90521954c 100644 --- a/arch/x86/include/asm/refcount.h +++ b/arch/x86/include/asm/refcount.h @@ -5,6 +5,7 @@ * PaX/grsecurity. */ #include <linux/refcount.h> +#include <asm/bug.h> /* * This is the first portion of the refcount error handling, which lives in diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h index 5c019d23d06b..4a911a382ade 100644 --- a/arch/x86/include/asm/sections.h +++ b/arch/x86/include/asm/sections.h @@ -7,6 +7,7 @@ extern char __brk_base[], __brk_limit[]; extern struct exception_table_entry __stop___ex_table[]; +extern char __end_rodata_aligned[]; #if defined(CONFIG_X86_64) extern char __end_rodata_hpage_align[]; diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index bd090367236c..34cffcef7375 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -46,6 +46,7 @@ int set_memory_np(unsigned long addr, int numpages); int set_memory_4k(unsigned long addr, int numpages); int set_memory_encrypted(unsigned long addr, int numpages); int set_memory_decrypted(unsigned long addr, int numpages); +int set_memory_np_noalias(unsigned long addr, int numpages); int set_memory_array_uc(unsigned long *addr, int addrinarray); int set_memory_array_wc(unsigned long *addr, int addrinarray); diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h index eb5f7999a893..36bd243843d6 100644 --- a/arch/x86/include/asm/switch_to.h +++ b/arch/x86/include/asm/switch_to.h @@ -87,15 +87,25 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread) #endif /* This is used when switching tasks or entering/exiting vm86 mode. */ -static inline void update_sp0(struct task_struct *task) +static inline void update_task_stack(struct task_struct *task) { - /* On x86_64, sp0 always points to the entry trampoline stack, which is constant: */ + /* sp0 always points to the entry trampoline stack, which is constant: */ #ifdef CONFIG_X86_32 - load_sp0(task->thread.sp0); + if (static_cpu_has(X86_FEATURE_XENPV)) + load_sp0(task->thread.sp0); + else + this_cpu_write(cpu_tss_rw.x86_tss.sp1, task->thread.sp0); #else + /* + * x86-64 updates x86_tss.sp1 via cpu_current_top_of_stack. That + * doesn't work on x86-32 because sp1 and + * cpu_current_top_of_stack have different values (because of + * the non-zero stack-padding on 32bit). + */ if (static_cpu_has(X86_FEATURE_XENPV)) load_sp0(task_top_of_stack(task)); #endif + } #endif /* _ASM_X86_SWITCH_TO_H */ diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 2ecd34e2d46c..e85ff65c43c3 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -37,5 +37,6 @@ extern void *text_poke_early(void *addr, const void *opcode, size_t len); extern void *text_poke(void *addr, const void *opcode, size_t len); extern int poke_int3_handler(struct pt_regs *regs); extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler); +extern int after_bootmem; #endif /* _ASM_X86_TEXT_PATCHING_H */ diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index 6690cd3fc8b1..511bf5fae8b8 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -148,22 +148,6 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) #define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr) #endif -static inline bool tlb_defer_switch_to_init_mm(void) -{ - /* - * If we have PCID, then switching to init_mm is reasonably - * fast. If we don't have PCID, then switching to init_mm is - * quite slow, so we try to defer it in the hopes that we can - * avoid it entirely. The latter approach runs the risk of - * receiving otherwise unnecessary IPIs. - * - * This choice is just a heuristic. The tlb code can handle this - * function returning true or false regardless of whether we have - * PCID. - */ - return !static_cpu_has(X86_FEATURE_PCID); -} - struct tlb_context { u64 ctx_id; u64 tlb_gen; @@ -554,4 +538,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); native_flush_tlb_others(mask, info) #endif +extern void tlb_flush_remove_tables(struct mm_struct *mm); +extern void tlb_flush_remove_tables_local(void *arg); + +#define HAVE_TLB_FLUSH_REMOVE_TABLES + #endif /* _ASM_X86_TLBFLUSH_H */ diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index c1d2a9892352..453cf38a1c33 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -123,13 +123,17 @@ static inline int topology_max_smt_threads(void) } int topology_update_package_map(unsigned int apicid, unsigned int cpu); -extern int topology_phys_to_logical_pkg(unsigned int pkg); +int topology_phys_to_logical_pkg(unsigned int pkg); +bool topology_is_primary_thread(unsigned int cpu); +bool topology_smt_supported(void); #else #define topology_max_packages() (1) static inline int topology_update_package_map(unsigned int apicid, unsigned int cpu) { return 0; } static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; } static inline int topology_max_smt_threads(void) { return 1; } +static inline bool topology_is_primary_thread(unsigned int cpu) { return true; } +static inline bool topology_smt_supported(void) { return false; } #endif static inline void arch_fix_phys_package_id(int num, u32 slot) diff --git a/arch/x86/include/asm/trace/hyperv.h b/arch/x86/include/asm/trace/hyperv.h index 4253bca99989..9c0d4b588e3f 100644 --- a/arch/x86/include/asm/trace/hyperv.h +++ b/arch/x86/include/asm/trace/hyperv.h @@ -28,6 +28,21 @@ TRACE_EVENT(hyperv_mmu_flush_tlb_others, __entry->addr, __entry->end) ); +TRACE_EVENT(hyperv_send_ipi_mask, + TP_PROTO(const struct cpumask *cpus, + int vector), + TP_ARGS(cpus, vector), + TP_STRUCT__entry( + __field(unsigned int, ncpus) + __field(int, vector) + ), + TP_fast_assign(__entry->ncpus = cpumask_weight(cpus); + __entry->vector = vector; + ), + TP_printk("ncpus %d vector %x", + __entry->ncpus, __entry->vector) + ); + #endif /* CONFIG_HYPERV */ #undef TRACE_INCLUDE_PATH diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index 2701d221583a..eb5bbfeccb66 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -33,13 +33,13 @@ static inline cycles_t get_cycles(void) extern struct system_counterval_t convert_art_to_tsc(u64 art); extern struct system_counterval_t convert_art_ns_to_tsc(u64 art_ns); -extern void tsc_early_delay_calibrate(void); +extern void tsc_early_init(void); extern void tsc_init(void); extern void mark_tsc_unstable(char *reason); extern int unsynchronized_tsc(void); extern int check_tsc_unstable(void); extern void mark_tsc_async_resets(char *reason); -extern unsigned long native_calibrate_cpu(void); +extern unsigned long native_calibrate_cpu_early(void); extern unsigned long native_calibrate_tsc(void); extern unsigned long long native_sched_clock_from_tsc(u64 tsc); diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 62acb613114b..a9d637bc301d 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -52,7 +52,12 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len) unsigned long ret; __uaccess_begin(); - ret = memcpy_mcsafe(to, from, len); + /* + * Note, __memcpy_mcsafe() is explicitly used since it can + * handle exceptions / faults. memcpy_mcsafe() may fall back to + * memcpy() which lacks this handling. + */ + ret = __memcpy_mcsafe(to, from, len); __uaccess_end(); return ret; } diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index bae46fc6b9de..0bcdb1279361 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -26,7 +26,7 @@ * the debuginfo as necessary. It will also warn if it sees any * inconsistencies. */ -.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL +.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL end=0 #ifdef CONFIG_STACK_VALIDATION .Lunwind_hint_ip_\@: .pushsection .discard.unwind_hints @@ -35,12 +35,14 @@ .short \sp_offset .byte \sp_reg .byte \type + .byte \end + .balign 4 .popsection #endif .endm .macro UNWIND_HINT_EMPTY - UNWIND_HINT sp_reg=ORC_REG_UNDEFINED + UNWIND_HINT sp_reg=ORC_REG_UNDEFINED end=1 .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0 @@ -86,19 +88,21 @@ #else /* !__ASSEMBLY__ */ -#define UNWIND_HINT(sp_reg, sp_offset, type) \ +#define UNWIND_HINT(sp_reg, sp_offset, type, end) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ ".long 987b - .\n\t" \ - ".short " __stringify(sp_offset) "\n\t" \ + ".short " __stringify(sp_offset) "\n\t" \ ".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(type) "\n\t" \ + ".byte " __stringify(end) "\n\t" \ + ".balign 4 \n\t" \ ".popsection\n\t" -#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE) +#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE, 0) -#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE) +#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE, 0) #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 6aa8499e1f62..95f9107449bf 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -576,4 +576,15 @@ enum vm_instruction_error_number { VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28, }; +enum vmx_l1d_flush_state { + VMENTER_L1D_FLUSH_AUTO, + VMENTER_L1D_FLUSH_NEVER, + VMENTER_L1D_FLUSH_COND, + VMENTER_L1D_FLUSH_ALWAYS, + VMENTER_L1D_FLUSH_EPT_DISABLED, + VMENTER_L1D_FLUSH_NOT_REQUIRED, +}; + +extern enum vmx_l1d_flush_state l1tf_vmx_mitigation; + #endif diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 02d6f5cf4e70..8824d01c0c35 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -61,6 +61,7 @@ obj-y += alternative.o i8253.o hw_breakpoint.o obj-y += tsc.o tsc_msr.o io_delay.o rtc.o obj-y += pci-iommu_table.o obj-y += resource.o +obj-y += irqflags.o obj-y += process.o obj-y += fpu/ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index a481763a3776..014f214da581 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -668,6 +668,7 @@ void *__init_or_module text_poke_early(void *addr, const void *opcode, local_irq_save(flags); memcpy(addr, opcode, len); local_irq_restore(flags); + sync_core(); /* Could also do a CLFLUSH here to speed up CPU recovery; but that causes hangs on some VIA CPUs. */ return addr; @@ -693,6 +694,12 @@ void *text_poke(void *addr, const void *opcode, size_t len) struct page *pages[2]; int i; + /* + * While boot memory allocator is runnig we cannot use struct + * pages as they are not yet initialized. + */ + BUG_ON(!after_bootmem); + if (!core_kernel_text((unsigned long)addr)) { pages[0] = vmalloc_to_page(addr); pages[1] = vmalloc_to_page(addr + PAGE_SIZE); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 2aabd4cb0e3f..87ff6235bbfe 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -56,6 +56,7 @@ #include <asm/hypervisor.h> #include <asm/cpu_device_id.h> #include <asm/intel-family.h> +#include <asm/irq_regs.h> unsigned int num_processors; @@ -573,6 +574,9 @@ static u32 skx_deadline_rev(void) case 0x04: return 0x02000014; } + if (boot_cpu_data.x86_stepping > 4) + return 0; + return ~0U; } @@ -937,7 +941,7 @@ static int __init calibrate_APIC_clock(void) if (levt->features & CLOCK_EVT_FEAT_DUMMY) { pr_warning("APIC timer disabled due to verification failure\n"); - return -1; + return -1; } return 0; @@ -2189,6 +2193,21 @@ static int cpuid_to_apicid[] = { [0 ... NR_CPUS - 1] = -1, }; +/** + * apic_id_is_primary_thread - Check whether APIC ID belongs to a primary thread + * @id: APIC ID to check + */ +bool apic_id_is_primary_thread(unsigned int apicid) +{ + u32 mask; + + if (smp_num_siblings == 1) + return true; + /* Isolate the SMT bit(s) in the APICID and check for 0 */ + mask = (1U << (fls(smp_num_siblings) - 1)) - 1; + return !(apicid & mask); +} + /* * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids * and cpuid_to_apicid[] synchronized. diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 3982f79d2377..ff0d14cd9e82 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -33,6 +33,7 @@ #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/init.h> #include <linux/delay.h> #include <linux/sched.h> diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index ce503c99f5c4..72a94401f9e0 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -12,6 +12,7 @@ */ #include <linux/mm.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/pci.h> #include <linux/dmar.h> #include <linux/hpet.h> diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 35aaee4fc028..9f148e3d45b4 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -11,6 +11,7 @@ * published by the Free Software Foundation. */ #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/seq_file.h> #include <linux/init.h> #include <linux/compiler.h> @@ -218,7 +219,8 @@ static int reserve_irq_vector(struct irq_data *irqd) return 0; } -static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest) +static int +assign_vector_locked(struct irq_data *irqd, const struct cpumask *dest) { struct apic_chip_data *apicd = apic_chip_data(irqd); bool resvd = apicd->has_reserved; @@ -245,22 +247,12 @@ static int allocate_vector(struct irq_data *irqd, const struct cpumask *dest) return -EBUSY; vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu); - if (vector > 0) - apic_update_vector(irqd, vector, cpu); trace_vector_alloc(irqd->irq, vector, resvd, vector); - return vector; -} - -static int assign_vector_locked(struct irq_data *irqd, - const struct cpumask *dest) -{ - struct apic_chip_data *apicd = apic_chip_data(irqd); - int vector = allocate_vector(irqd, dest); - if (vector < 0) return vector; + apic_update_vector(irqd, vector, cpu); + apic_update_irq_cfg(irqd, vector, cpu); - apic_update_irq_cfg(irqd, apicd->vector, apicd->cpu); return 0; } @@ -433,7 +425,7 @@ static int activate_managed(struct irq_data *irqd) pr_err("Managed startup irq %u, no vector available\n", irqd->irq); } - return ret; + return ret; } static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd, diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index d492752f79e1..391f358ebb4c 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -394,10 +394,10 @@ extern int uv_hub_info_version(void) EXPORT_SYMBOL(uv_hub_info_version); /* Default UV memory block size is 2GB */ -static unsigned long mem_block_size = (2UL << 30); +static unsigned long mem_block_size __initdata = (2UL << 30); /* Kernel parameter to specify UV mem block size */ -static int parse_mem_block_size(char *ptr) +static int __init parse_mem_block_size(char *ptr) { unsigned long size = memparse(ptr, NULL); diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index 5d0de79fdab0..ec00d1ff5098 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -240,6 +240,7 @@ #include <asm/olpc.h> #include <asm/paravirt.h> #include <asm/reboot.h> +#include <asm/nospec-branch.h> #if defined(CONFIG_APM_DISPLAY_BLANK) && defined(CONFIG_VT) extern int (*console_blank_hook)(int); @@ -614,11 +615,13 @@ static long __apm_bios_call(void *_call) gdt[0x40 / 8] = bad_bios_desc; apm_irq_save(flags); + firmware_restrict_branch_speculation_start(); APM_DO_SAVE_SEGS; apm_bios_call_asm(call->func, call->ebx, call->ecx, &call->eax, &call->ebx, &call->ecx, &call->edx, &call->esi); APM_DO_RESTORE_SEGS; + firmware_restrict_branch_speculation_end(); apm_irq_restore(flags); gdt[0x40 / 8] = save_desc_40; put_cpu(); @@ -690,10 +693,12 @@ static long __apm_bios_call_simple(void *_call) gdt[0x40 / 8] = bad_bios_desc; apm_irq_save(flags); + firmware_restrict_branch_speculation_start(); APM_DO_SAVE_SEGS; error = apm_bios_call_simple_asm(call->func, call->ebx, call->ecx, &call->eax); APM_DO_RESTORE_SEGS; + firmware_restrict_branch_speculation_end(); apm_irq_restore(flags); gdt[0x40 / 8] = save_desc_40; put_cpu(); diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index dcb008c320fe..01de31db300d 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -103,4 +103,9 @@ void common(void) { OFFSET(CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline); OFFSET(CPU_ENTRY_AREA_entry_stack, cpu_entry_area, entry_stack_page); DEFINE(SIZEOF_entry_stack, sizeof(struct entry_stack)); + DEFINE(MASK_entry_stack, (~(sizeof(struct entry_stack) - 1))); + + /* Offset for sp0 and sp1 into the tss_struct */ + OFFSET(TSS_sp0, tss_struct, x86_tss.sp0); + OFFSET(TSS_sp1, tss_struct, x86_tss.sp1); } diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index a4a3be399f4b..82826f2275cc 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c @@ -46,8 +46,14 @@ void foo(void) OFFSET(saved_context_gdt_desc, saved_context, gdt_desc); BLANK(); - /* Offset from the sysenter stack to tss.sp0 */ - DEFINE(TSS_sysenter_sp0, offsetof(struct cpu_entry_area, tss.x86_tss.sp0) - + /* + * Offset from the entry stack to task stack stored in TSS. Kernel entry + * happens on the per-cpu entry-stack, and the asm code switches to the + * task-stack pointer stored in x86_tss.sp1, which is a copy of + * task->thread.sp0 where entry code can find it. + */ + DEFINE(TSS_entry2task_stack, + offsetof(struct cpu_entry_area, tss.x86_tss.sp1) - offsetofend(struct cpu_entry_area, entry_stack_page.stack)); #ifdef CONFIG_STACKPROTECTOR diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index b2dcd161f514..3b9405e7ba2b 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -65,8 +65,6 @@ int main(void) #undef ENTRY OFFSET(TSS_ist, tss_struct, x86_tss.ist); - OFFSET(TSS_sp0, tss_struct, x86_tss.sp0); - OFFSET(TSS_sp1, tss_struct, x86_tss.sp1); BLANK(); #ifdef CONFIG_STACKPROTECTOR diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 7a40196967cb..347137e80bf5 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -35,7 +35,9 @@ obj-$(CONFIG_CPU_SUP_CENTAUR) += centaur.o obj-$(CONFIG_CPU_SUP_TRANSMETA_32) += transmeta.o obj-$(CONFIG_CPU_SUP_UMC_32) += umc.o -obj-$(CONFIG_INTEL_RDT) += intel_rdt.o intel_rdt_rdtgroup.o intel_rdt_monitor.o intel_rdt_ctrlmondata.o +obj-$(CONFIG_INTEL_RDT) += intel_rdt.o intel_rdt_rdtgroup.o intel_rdt_monitor.o +obj-$(CONFIG_INTEL_RDT) += intel_rdt_ctrlmondata.o intel_rdt_pseudo_lock.o +CFLAGS_intel_rdt_pseudo_lock.o = -I$(src) obj-$(CONFIG_X86_MCE) += mcheck/ obj-$(CONFIG_MTRR) += mtrr/ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 082d7875cef8..22ab408177b2 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -232,8 +232,6 @@ static void init_amd_k7(struct cpuinfo_x86 *c) } } - set_cpu_cap(c, X86_FEATURE_K7); - /* calling is from identify_secondary_cpu() ? */ if (!c->cpu_index) return; @@ -315,6 +313,13 @@ static void legacy_fixup_core_id(struct cpuinfo_x86 *c) c->cpu_core_id %= cus_per_node; } + +static void amd_get_topology_early(struct cpuinfo_x86 *c) +{ + if (cpu_has(c, X86_FEATURE_TOPOEXT)) + smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1; +} + /* * Fixup core topology information for * (1) AMD multi-node processors @@ -334,7 +339,6 @@ static void amd_get_topology(struct cpuinfo_x86 *c) cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); node_id = ecx & 0xff; - smp_num_siblings = ((ebx >> 8) & 0xff) + 1; if (c->x86 == 0x15) c->cu_id = ebx & 0xff; @@ -543,7 +547,9 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) nodes_per_socket = ((value >> 3) & 7) + 1; } - if (c->x86 >= 0x15 && c->x86 <= 0x17) { + if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) && + !boot_cpu_has(X86_FEATURE_VIRT_SSBD) && + c->x86 >= 0x15 && c->x86 <= 0x17) { unsigned int bit; switch (c->x86) { @@ -611,10 +617,19 @@ clear_sev: static void early_init_amd(struct cpuinfo_x86 *c) { + u64 value; u32 dummy; early_init_amd_mc(c); +#ifdef CONFIG_X86_32 + if (c->x86 == 6) + set_cpu_cap(c, X86_FEATURE_K7); +#endif + + if (c->x86 >= 0xf) + set_cpu_cap(c, X86_FEATURE_K8); + rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy); /* @@ -681,6 +696,22 @@ static void early_init_amd(struct cpuinfo_x86 *c) set_cpu_bug(c, X86_BUG_AMD_E400); early_detect_mem_encrypt(c); + + /* Re-enable TopologyExtensions if switched off by BIOS */ + if (c->x86 == 0x15 && + (c->x86_model >= 0x10 && c->x86_model <= 0x6f) && + !cpu_has(c, X86_FEATURE_TOPOEXT)) { + + if (msr_set_bit(0xc0011005, 54) > 0) { + rdmsrl(0xc0011005, value); + if (value & BIT_64(54)) { + set_cpu_cap(c, X86_FEATURE_TOPOEXT); + pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n"); + } + } + } + + amd_get_topology_early(c); } static void init_amd_k8(struct cpuinfo_x86 *c) @@ -772,19 +803,6 @@ static void init_amd_bd(struct cpuinfo_x86 *c) { u64 value; - /* re-enable TopologyExtensions if switched off by BIOS */ - if ((c->x86_model >= 0x10) && (c->x86_model <= 0x6f) && - !cpu_has(c, X86_FEATURE_TOPOEXT)) { - - if (msr_set_bit(0xc0011005, 54) > 0) { - rdmsrl(0xc0011005, value); - if (value & BIT_64(54)) { - set_cpu_cap(c, X86_FEATURE_TOPOEXT); - pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n"); - } - } - } - /* * The way access filter has a performance penalty on some workloads. * Disable it on the affected CPUs. @@ -848,22 +866,12 @@ static void init_amd(struct cpuinfo_x86 *c) cpu_detect_cache_sizes(c); - /* Multi core CPU? */ - if (c->extended_cpuid_level >= 0x80000008) { - amd_detect_cmp(c); - amd_get_topology(c); - srat_detect_node(c); - } - -#ifdef CONFIG_X86_32 - detect_ht(c); -#endif + amd_detect_cmp(c); + amd_get_topology(c); + srat_detect_node(c); init_amd_cacheinfo(c); - if (c->x86 >= 0xf) - set_cpu_cap(c, X86_FEATURE_K8); - if (cpu_has(c, X86_FEATURE_XMM2)) { unsigned long long val; int ret; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 404df26b7de8..27830880e7a7 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -22,15 +22,18 @@ #include <asm/processor-flags.h> #include <asm/fpu/internal.h> #include <asm/msr.h> +#include <asm/vmx.h> #include <asm/paravirt.h> #include <asm/alternative.h> #include <asm/pgtable.h> #include <asm/set_memory.h> #include <asm/intel-family.h> +#include <asm/e820/api.h> #include <asm/hypervisor.h> static void __init spectre_v2_select_mitigation(void); static void __init ssb_select_mitigation(void); +static void __init l1tf_select_mitigation(void); /* * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any @@ -56,6 +59,12 @@ void __init check_bugs(void) { identify_boot_cpu(); + /* + * identify_boot_cpu() initialized SMT support information, let the + * core code know. + */ + cpu_smt_check_topology_early(); + if (!IS_ENABLED(CONFIG_SMP)) { pr_info("CPU: "); print_cpu_info(&boot_cpu_data); @@ -82,6 +91,8 @@ void __init check_bugs(void) */ ssb_select_mitigation(); + l1tf_select_mitigation(); + #ifdef CONFIG_X86_32 /* * Check whether we are able to run this kernel safely on SMP. @@ -130,6 +141,7 @@ static const char *spectre_v2_strings[] = { [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline", [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", + [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", }; #undef pr_fmt @@ -155,7 +167,8 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) guestval |= guest_spec_ctrl & x86_spec_ctrl_mask; /* SSBD controlled in MSR_SPEC_CTRL */ - if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD)) + if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || + static_cpu_has(X86_FEATURE_AMD_SSBD)) hostval |= ssbd_tif_to_spec_ctrl(ti->flags); if (hostval != guestval) { @@ -312,23 +325,6 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) return cmd; } -/* Check for Skylake-like CPUs (for RSB handling) */ -static bool __init is_skylake_era(void) -{ - if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && - boot_cpu_data.x86 == 6) { - switch (boot_cpu_data.x86_model) { - case INTEL_FAM6_SKYLAKE_MOBILE: - case INTEL_FAM6_SKYLAKE_DESKTOP: - case INTEL_FAM6_SKYLAKE_X: - case INTEL_FAM6_KABYLAKE_MOBILE: - case INTEL_FAM6_KABYLAKE_DESKTOP: - return true; - } - } - return false; -} - static void __init spectre_v2_select_mitigation(void) { enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); @@ -348,6 +344,13 @@ static void __init spectre_v2_select_mitigation(void) case SPECTRE_V2_CMD_FORCE: case SPECTRE_V2_CMD_AUTO: + if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { + mode = SPECTRE_V2_IBRS_ENHANCED; + /* Force it so VMEXIT will restore correctly */ + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); + goto specv2_set_mode; + } if (IS_ENABLED(CONFIG_RETPOLINE)) goto retpoline_auto; break; @@ -385,26 +388,20 @@ retpoline_auto: setup_force_cpu_cap(X86_FEATURE_RETPOLINE); } +specv2_set_mode: spectre_v2_enabled = mode; pr_info("%s\n", spectre_v2_strings[mode]); /* - * If neither SMEP nor PTI are available, there is a risk of - * hitting userspace addresses in the RSB after a context switch - * from a shallow call stack to a deeper one. To prevent this fill - * the entire RSB, even when using IBRS. + * If spectre v2 protection has been enabled, unconditionally fill + * RSB during a context switch; this protects against two independent + * issues: * - * Skylake era CPUs have a separate issue with *underflow* of the - * RSB, when they will predict 'ret' targets from the generic BTB. - * The proper mitigation for this is IBRS. If IBRS is not supported - * or deactivated in favour of retpolines the RSB fill on context - * switch is required. + * - RSB underflow (and switch to BTB) on Skylake+ + * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs */ - if ((!boot_cpu_has(X86_FEATURE_PTI) && - !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) { - setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); - pr_info("Spectre v2 mitigation: Filling RSB on context switch\n"); - } + setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); + pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); /* Initialize Indirect Branch Prediction Barrier if supported */ if (boot_cpu_has(X86_FEATURE_IBPB)) { @@ -414,9 +411,16 @@ retpoline_auto: /* * Retpoline means the kernel is safe because it has no indirect - * branches. But firmware isn't, so use IBRS to protect that. + * branches. Enhanced IBRS protects firmware too, so, enable restricted + * speculation around firmware calls only when Enhanced IBRS isn't + * supported. + * + * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because + * the user might select retpoline on the kernel command line and if + * the CPU supports Enhanced IBRS, kernel might un-intentionally not + * enable IBRS around firmware calls. */ - if (boot_cpu_has(X86_FEATURE_IBRS)) { + if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } @@ -533,9 +537,10 @@ static enum ssb_mitigation __init __ssb_select_mitigation(void) * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may * use a completely different MSR and bit dependent on family. */ - if (!static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) + if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && + !static_cpu_has(X86_FEATURE_AMD_SSBD)) { x86_amd_ssb_disable(); - else { + } else { x86_spec_ctrl_base |= SPEC_CTRL_SSBD; x86_spec_ctrl_mask |= SPEC_CTRL_SSBD; wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); @@ -652,8 +657,121 @@ void x86_spec_ctrl_setup_ap(void) x86_amd_ssb_disable(); } +#undef pr_fmt +#define pr_fmt(fmt) "L1TF: " fmt + +/* Default mitigation for L1TF-affected CPUs */ +enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; +#if IS_ENABLED(CONFIG_KVM_INTEL) +EXPORT_SYMBOL_GPL(l1tf_mitigation); + +enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; +EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); +#endif + +static void __init l1tf_select_mitigation(void) +{ + u64 half_pa; + + if (!boot_cpu_has_bug(X86_BUG_L1TF)) + return; + + switch (l1tf_mitigation) { + case L1TF_MITIGATION_OFF: + case L1TF_MITIGATION_FLUSH_NOWARN: + case L1TF_MITIGATION_FLUSH: + break; + case L1TF_MITIGATION_FLUSH_NOSMT: + case L1TF_MITIGATION_FULL: + cpu_smt_disable(false); + break; + case L1TF_MITIGATION_FULL_FORCE: + cpu_smt_disable(true); + break; + } + +#if CONFIG_PGTABLE_LEVELS == 2 + pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); + return; +#endif + + /* + * This is extremely unlikely to happen because almost all + * systems have far more MAX_PA/2 than RAM can be fit into + * DIMM slots. + */ + half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; + if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { + pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); + return; + } + + setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); +} + +static int __init l1tf_cmdline(char *str) +{ + if (!boot_cpu_has_bug(X86_BUG_L1TF)) + return 0; + + if (!str) + return -EINVAL; + + if (!strcmp(str, "off")) + l1tf_mitigation = L1TF_MITIGATION_OFF; + else if (!strcmp(str, "flush,nowarn")) + l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; + else if (!strcmp(str, "flush")) + l1tf_mitigation = L1TF_MITIGATION_FLUSH; + else if (!strcmp(str, "flush,nosmt")) + l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; + else if (!strcmp(str, "full")) + l1tf_mitigation = L1TF_MITIGATION_FULL; + else if (!strcmp(str, "full,force")) + l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; + + return 0; +} +early_param("l1tf", l1tf_cmdline); + +#undef pr_fmt + #ifdef CONFIG_SYSFS +#define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" + +#if IS_ENABLED(CONFIG_KVM_INTEL) +static const char *l1tf_vmx_states[] = { + [VMENTER_L1D_FLUSH_AUTO] = "auto", + [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", + [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", + [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", + [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", + [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" +}; + +static ssize_t l1tf_show_state(char *buf) +{ + if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) + return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); + + if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || + (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && + cpu_smt_control == CPU_SMT_ENABLED)) + return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, + l1tf_vmx_states[l1tf_vmx_mitigation]); + + return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, + l1tf_vmx_states[l1tf_vmx_mitigation], + cpu_smt_control == CPU_SMT_ENABLED ? "vulnerable" : "disabled"); +} +#else +static ssize_t l1tf_show_state(char *buf) +{ + return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); +} +#endif + static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, char *buf, unsigned int bug) { @@ -682,6 +800,10 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr case X86_BUG_SPEC_STORE_BYPASS: return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); + case X86_BUG_L1TF: + if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) + return l1tf_show_state(buf); + break; default: break; } @@ -708,4 +830,9 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute * { return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); } + +ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) +{ + return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); +} #endif diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index eb4cb3efd20e..f3234010847c 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -661,33 +661,36 @@ static void cpu_detect_tlb(struct cpuinfo_x86 *c) tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]); } -void detect_ht(struct cpuinfo_x86 *c) +int detect_ht_early(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP u32 eax, ebx, ecx, edx; - int index_msb, core_bits; - static bool printed; if (!cpu_has(c, X86_FEATURE_HT)) - return; + return -1; if (cpu_has(c, X86_FEATURE_CMP_LEGACY)) - goto out; + return -1; if (cpu_has(c, X86_FEATURE_XTOPOLOGY)) - return; + return -1; cpuid(1, &eax, &ebx, &ecx, &edx); smp_num_siblings = (ebx & 0xff0000) >> 16; - - if (smp_num_siblings == 1) { + if (smp_num_siblings == 1) pr_info_once("CPU0: Hyper-Threading is disabled\n"); - goto out; - } +#endif + return 0; +} - if (smp_num_siblings <= 1) - goto out; +void detect_ht(struct cpuinfo_x86 *c) +{ +#ifdef CONFIG_SMP + int index_msb, core_bits; + + if (detect_ht_early(c) < 0) + return; index_msb = get_count_order(smp_num_siblings); c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb); @@ -700,15 +703,6 @@ void detect_ht(struct cpuinfo_x86 *c) c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) & ((1 << core_bits) - 1); - -out: - if (!printed && (c->x86_max_cores * smp_num_siblings) > 1) { - pr_info("CPU: Physical Processor ID: %d\n", - c->phys_proc_id); - pr_info("CPU: Processor Core ID: %d\n", - c->cpu_core_id); - printed = 1; - } #endif } @@ -987,6 +981,21 @@ static const __initconst struct x86_cpu_id cpu_no_spec_store_bypass[] = { {} }; +static const __initconst struct x86_cpu_id cpu_no_l1tf[] = { + /* in addition to cpu_no_speculation */ + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT1 }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT2 }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_AIRMONT }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_MERRIFIELD }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_MOOREFIELD }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GOLDMONT }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_DENVERTON }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_GEMINI_LAKE }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_XEON_PHI_KNL }, + { X86_VENDOR_INTEL, 6, INTEL_FAM6_XEON_PHI_KNM }, + {} +}; + static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) { u64 ia32_cap = 0; @@ -1005,6 +1014,9 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) !cpu_has(c, X86_FEATURE_AMD_SSB_NO)) setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS); + if (ia32_cap & ARCH_CAP_IBRS_ALL) + setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED); + if (x86_match_cpu(cpu_no_meltdown)) return; @@ -1013,6 +1025,29 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) return; setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN); + + if (x86_match_cpu(cpu_no_l1tf)) + return; + + setup_force_cpu_bug(X86_BUG_L1TF); +} + +/* + * The NOPL instruction is supposed to exist on all CPUs of family >= 6; + * unfortunately, that's not true in practice because of early VIA + * chips and (more importantly) broken virtualizers that are not easy + * to detect. In the latter case it doesn't even *fail* reliably, so + * probing for it doesn't even work. Disable it completely on 32-bit + * unless we can find a reliable way to detect all the broken cases. + * Enable it explicitly on 64-bit for non-constant inputs of cpu_has(). + */ +static void detect_nopl(void) +{ +#ifdef CONFIG_X86_32 + setup_clear_cpu_cap(X86_FEATURE_NOPL); +#else + setup_force_cpu_cap(X86_FEATURE_NOPL); +#endif } /* @@ -1089,6 +1124,8 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) */ if (!pgtable_l5_enabled()) setup_clear_cpu_cap(X86_FEATURE_LA57); + + detect_nopl(); } void __init early_cpu_init(void) @@ -1124,24 +1161,6 @@ void __init early_cpu_init(void) early_identify_cpu(&boot_cpu_data); } -/* - * The NOPL instruction is supposed to exist on all CPUs of family >= 6; - * unfortunately, that's not true in practice because of early VIA - * chips and (more importantly) broken virtualizers that are not easy - * to detect. In the latter case it doesn't even *fail* reliably, so - * probing for it doesn't even work. Disable it completely on 32-bit - * unless we can find a reliable way to detect all the broken cases. - * Enable it explicitly on 64-bit for non-constant inputs of cpu_has(). - */ -static void detect_nopl(struct cpuinfo_x86 *c) -{ -#ifdef CONFIG_X86_32 - clear_cpu_cap(c, X86_FEATURE_NOPL); -#else - set_cpu_cap(c, X86_FEATURE_NOPL); -#endif -} - static void detect_null_seg_behavior(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_64 @@ -1204,8 +1223,6 @@ static void generic_identify(struct cpuinfo_x86 *c) get_model_name(c); /* Default name */ - detect_nopl(c); - detect_null_seg_behavior(c); /* @@ -1804,11 +1821,12 @@ void cpu_init(void) enter_lazy_tlb(&init_mm, curr); /* - * Initialize the TSS. Don't bother initializing sp0, as the initial - * task never enters user mode. + * Initialize the TSS. sp0 points to the entry trampoline stack + * regardless of what task is running. */ set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss); load_TR_desc(); + load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1)); load_mm_ldt(&init_mm); diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 38216f678fc3..e59c0ea82a33 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -55,7 +55,9 @@ extern void init_intel_cacheinfo(struct cpuinfo_x86 *c); extern void init_amd_cacheinfo(struct cpuinfo_x86 *c); extern void detect_num_cpu_cores(struct cpuinfo_x86 *c); +extern int detect_extended_topology_early(struct cpuinfo_x86 *c); extern int detect_extended_topology(struct cpuinfo_x86 *c); +extern int detect_ht_early(struct cpuinfo_x86 *c); extern void detect_ht(struct cpuinfo_x86 *c); unsigned int aperfmperf_get_khz(int cpu); diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index eb75564f2d25..401e8c133108 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -301,6 +301,13 @@ static void early_init_intel(struct cpuinfo_x86 *c) } check_mpx_erratum(c); + + /* + * Get the number of SMT siblings early from the extended topology + * leaf, if available. Otherwise try the legacy SMT detection. + */ + if (detect_extended_topology_early(c) < 0) + detect_ht_early(c); } #ifdef CONFIG_X86_32 @@ -465,14 +472,17 @@ static void detect_vmx_virtcap(struct cpuinfo_x86 *c) #define X86_VMX_FEATURE_PROC_CTLS2_VIRT_APIC 0x00000001 #define X86_VMX_FEATURE_PROC_CTLS2_EPT 0x00000002 #define X86_VMX_FEATURE_PROC_CTLS2_VPID 0x00000020 +#define x86_VMX_FEATURE_EPT_CAP_AD 0x00200000 u32 vmx_msr_low, vmx_msr_high, msr_ctl, msr_ctl2; + u32 msr_vpid_cap, msr_ept_cap; clear_cpu_cap(c, X86_FEATURE_TPR_SHADOW); clear_cpu_cap(c, X86_FEATURE_VNMI); clear_cpu_cap(c, X86_FEATURE_FLEXPRIORITY); clear_cpu_cap(c, X86_FEATURE_EPT); clear_cpu_cap(c, X86_FEATURE_VPID); + clear_cpu_cap(c, X86_FEATURE_EPT_AD); rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, vmx_msr_low, vmx_msr_high); msr_ctl = vmx_msr_high | vmx_msr_low; @@ -487,8 +497,13 @@ static void detect_vmx_virtcap(struct cpuinfo_x86 *c) if ((msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_VIRT_APIC) && (msr_ctl & X86_VMX_FEATURE_PROC_CTLS_TPR_SHADOW)) set_cpu_cap(c, X86_FEATURE_FLEXPRIORITY); - if (msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_EPT) + if (msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_EPT) { set_cpu_cap(c, X86_FEATURE_EPT); + rdmsr(MSR_IA32_VMX_EPT_VPID_CAP, + msr_ept_cap, msr_vpid_cap); + if (msr_ept_cap & x86_VMX_FEATURE_EPT_CAP_AD) + set_cpu_cap(c, X86_FEATURE_EPT_AD); + } if (msr_ctl2 & X86_VMX_FEATURE_PROC_CTLS2_VPID) set_cpu_cap(c, X86_FEATURE_VPID); } diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c index ec4754f81cbd..abb71ac70443 100644 --- a/arch/x86/kernel/cpu/intel_rdt.c +++ b/arch/x86/kernel/cpu/intel_rdt.c @@ -859,6 +859,8 @@ static __init bool get_rdt_resources(void) return (rdt_mon_capable || rdt_alloc_capable); } +static enum cpuhp_state rdt_online; + static int __init intel_rdt_late_init(void) { struct rdt_resource *r; @@ -880,6 +882,7 @@ static int __init intel_rdt_late_init(void) cpuhp_remove_state(state); return ret; } + rdt_online = state; for_each_alloc_capable_rdt_resource(r) pr_info("Intel RDT %s allocation detected\n", r->name); @@ -891,3 +894,11 @@ static int __init intel_rdt_late_init(void) } late_initcall(intel_rdt_late_init); + +static void __exit intel_rdt_exit(void) +{ + cpuhp_remove_state(rdt_online); + rdtgroup_exit(); +} + +__exitcall(intel_rdt_exit); diff --git a/arch/x86/kernel/cpu/intel_rdt.h b/arch/x86/kernel/cpu/intel_rdt.h index 39752825e376..4e588f36228f 100644 --- a/arch/x86/kernel/cpu/intel_rdt.h +++ b/arch/x86/kernel/cpu/intel_rdt.h @@ -81,6 +81,34 @@ enum rdt_group_type { }; /** + * enum rdtgrp_mode - Mode of a RDT resource group + * @RDT_MODE_SHAREABLE: This resource group allows sharing of its allocations + * @RDT_MODE_EXCLUSIVE: No sharing of this resource group's allocations allowed + * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking + * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations + * allowed AND the allocations are Cache Pseudo-Locked + * + * The mode of a resource group enables control over the allowed overlap + * between allocations associated with different resource groups (classes + * of service). User is able to modify the mode of a resource group by + * writing to the "mode" resctrl file associated with the resource group. + * + * The "shareable", "exclusive", and "pseudo-locksetup" modes are set by + * writing the appropriate text to the "mode" file. A resource group enters + * "pseudo-locked" mode after the schemata is written while the resource + * group is in "pseudo-locksetup" mode. + */ +enum rdtgrp_mode { + RDT_MODE_SHAREABLE = 0, + RDT_MODE_EXCLUSIVE, + RDT_MODE_PSEUDO_LOCKSETUP, + RDT_MODE_PSEUDO_LOCKED, + + /* Must be last */ + RDT_NUM_MODES, +}; + +/** * struct mongroup - store mon group's data in resctrl fs. * @mon_data_kn kernlfs node for the mon_data directory * @parent: parent rdtgrp @@ -95,6 +123,43 @@ struct mongroup { }; /** + * struct pseudo_lock_region - pseudo-lock region information + * @r: RDT resource to which this pseudo-locked region + * belongs + * @d: RDT domain to which this pseudo-locked region + * belongs + * @cbm: bitmask of the pseudo-locked region + * @lock_thread_wq: waitqueue used to wait on the pseudo-locking thread + * completion + * @thread_done: variable used by waitqueue to test if pseudo-locking + * thread completed + * @cpu: core associated with the cache on which the setup code + * will be run + * @line_size: size of the cache lines + * @size: size of pseudo-locked region in bytes + * @kmem: the kernel memory associated with pseudo-locked region + * @minor: minor number of character device associated with this + * region + * @debugfs_dir: pointer to this region's directory in the debugfs + * filesystem + * @pm_reqs: Power management QoS requests related to this region + */ +struct pseudo_lock_region { + struct rdt_resource *r; + struct rdt_domain *d; + u32 cbm; + wait_queue_head_t lock_thread_wq; + int thread_done; + int cpu; + unsigned int line_size; + unsigned int size; + void *kmem; + unsigned int minor; + struct dentry *debugfs_dir; + struct list_head pm_reqs; +}; + +/** * struct rdtgroup - store rdtgroup's data in resctrl file system. * @kn: kernfs node * @rdtgroup_list: linked list for all rdtgroups @@ -106,16 +171,20 @@ struct mongroup { * @type: indicates type of this rdtgroup - either * monitor only or ctrl_mon group * @mon: mongroup related data + * @mode: mode of resource group + * @plr: pseudo-locked region */ struct rdtgroup { - struct kernfs_node *kn; - struct list_head rdtgroup_list; - u32 closid; - struct cpumask cpu_mask; - int flags; - atomic_t waitcount; - enum rdt_group_type type; - struct mongroup mon; + struct kernfs_node *kn; + struct list_head rdtgroup_list; + u32 closid; + struct cpumask cpu_mask; + int flags; + atomic_t waitcount; + enum rdt_group_type type; + struct mongroup mon; + enum rdtgrp_mode mode; + struct pseudo_lock_region *plr; }; /* rdtgroup.flags */ @@ -148,6 +217,7 @@ extern struct list_head rdt_all_groups; extern int max_name_width, max_data_width; int __init rdtgroup_init(void); +void __exit rdtgroup_exit(void); /** * struct rftype - describe each file in the resctrl file system @@ -216,22 +286,24 @@ struct mbm_state { * @mbps_val: When mba_sc is enabled, this holds the bandwidth in MBps * @new_ctrl: new ctrl value to be loaded * @have_new_ctrl: did user provide new_ctrl for this domain + * @plr: pseudo-locked region (if any) associated with domain */ struct rdt_domain { - struct list_head list; - int id; - struct cpumask cpu_mask; - unsigned long *rmid_busy_llc; - struct mbm_state *mbm_total; - struct mbm_state *mbm_local; - struct delayed_work mbm_over; - struct delayed_work cqm_limbo; - int mbm_work_cpu; - int cqm_work_cpu; - u32 *ctrl_val; - u32 *mbps_val; - u32 new_ctrl; - bool have_new_ctrl; + struct list_head list; + int id; + struct cpumask cpu_mask; + unsigned long *rmid_busy_llc; + struct mbm_state *mbm_total; + struct mbm_state *mbm_local; + struct delayed_work mbm_over; + struct delayed_work cqm_limbo; + int mbm_work_cpu; + int cqm_work_cpu; + u32 *ctrl_val; + u32 *mbps_val; + u32 new_ctrl; + bool have_new_ctrl; + struct pseudo_lock_region *plr; }; /** @@ -351,7 +423,7 @@ struct rdt_resource { struct rdt_cache cache; struct rdt_membw membw; const char *format_str; - int (*parse_ctrlval) (char *buf, struct rdt_resource *r, + int (*parse_ctrlval) (void *data, struct rdt_resource *r, struct rdt_domain *d); struct list_head evt_list; int num_rmid; @@ -359,8 +431,8 @@ struct rdt_resource { unsigned long fflags; }; -int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d); -int parse_bw(char *buf, struct rdt_resource *r, struct rdt_domain *d); +int parse_cbm(void *_data, struct rdt_resource *r, struct rdt_domain *d); +int parse_bw(void *_buf, struct rdt_resource *r, struct rdt_domain *d); extern struct mutex rdtgroup_mutex; @@ -368,7 +440,7 @@ extern struct rdt_resource rdt_resources_all[]; extern struct rdtgroup rdtgroup_default; DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key); -int __init rdtgroup_init(void); +extern struct dentry *debugfs_resctrl; enum { RDT_RESOURCE_L3, @@ -439,13 +511,32 @@ void rdt_last_cmd_printf(const char *fmt, ...); void rdt_ctrl_update(void *arg); struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); void rdtgroup_kn_unlock(struct kernfs_node *kn); +int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name); +int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, + umode_t mask); struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, struct list_head **pos); ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off); int rdtgroup_schemata_show(struct kernfs_open_file *of, struct seq_file *s, void *v); +bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d, + u32 _cbm, int closid, bool exclusive); +unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_domain *d, + u32 cbm); +enum rdtgrp_mode rdtgroup_mode_by_closid(int closid); +int rdtgroup_tasks_assigned(struct rdtgroup *r); +int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp); +int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp); +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, u32 _cbm); +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d); +int rdt_pseudo_lock_init(void); +void rdt_pseudo_lock_release(void); +int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp); +void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp); struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r); +int update_domains(struct rdt_resource *r, int closid); +void closid_free(int closid); int alloc_rmid(void); void free_rmid(u32 rmid); int rdt_get_mon_l3_config(struct rdt_resource *r); diff --git a/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c b/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c index 116d57b248d3..af358ca05160 100644 --- a/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c +++ b/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c @@ -64,9 +64,10 @@ static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r) return true; } -int parse_bw(char *buf, struct rdt_resource *r, struct rdt_domain *d) +int parse_bw(void *_buf, struct rdt_resource *r, struct rdt_domain *d) { unsigned long data; + char *buf = _buf; if (d->have_new_ctrl) { rdt_last_cmd_printf("duplicate domain %d\n", d->id); @@ -87,7 +88,7 @@ int parse_bw(char *buf, struct rdt_resource *r, struct rdt_domain *d) * are allowed (e.g. FFFFH, 0FF0H, 003CH, etc.). * Additionally Haswell requires at least two bits set. */ -static bool cbm_validate(char *buf, unsigned long *data, struct rdt_resource *r) +static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) { unsigned long first_bit, zero_bit, val; unsigned int cbm_len = r->cache.cbm_len; @@ -122,22 +123,64 @@ static bool cbm_validate(char *buf, unsigned long *data, struct rdt_resource *r) return true; } +struct rdt_cbm_parse_data { + struct rdtgroup *rdtgrp; + char *buf; +}; + /* * Read one cache bit mask (hex). Check that it is valid for the current * resource type. */ -int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d) +int parse_cbm(void *_data, struct rdt_resource *r, struct rdt_domain *d) { - unsigned long data; + struct rdt_cbm_parse_data *data = _data; + struct rdtgroup *rdtgrp = data->rdtgrp; + u32 cbm_val; if (d->have_new_ctrl) { rdt_last_cmd_printf("duplicate domain %d\n", d->id); return -EINVAL; } - if(!cbm_validate(buf, &data, r)) + /* + * Cannot set up more than one pseudo-locked region in a cache + * hierarchy. + */ + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && + rdtgroup_pseudo_locked_in_hierarchy(d)) { + rdt_last_cmd_printf("pseudo-locked region in hierarchy\n"); return -EINVAL; - d->new_ctrl = data; + } + + if (!cbm_validate(data->buf, &cbm_val, r)) + return -EINVAL; + + if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE || + rdtgrp->mode == RDT_MODE_SHAREABLE) && + rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) { + rdt_last_cmd_printf("CBM overlaps with pseudo-locked region\n"); + return -EINVAL; + } + + /* + * The CBM may not overlap with the CBM of another closid if + * either is exclusive. + */ + if (rdtgroup_cbm_overlaps(r, d, cbm_val, rdtgrp->closid, true)) { + rdt_last_cmd_printf("overlaps with exclusive group\n"); + return -EINVAL; + } + + if (rdtgroup_cbm_overlaps(r, d, cbm_val, rdtgrp->closid, false)) { + if (rdtgrp->mode == RDT_MODE_EXCLUSIVE || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + rdt_last_cmd_printf("overlaps with other group\n"); + return -EINVAL; + } + } + + d->new_ctrl = cbm_val; d->have_new_ctrl = true; return 0; @@ -149,8 +192,10 @@ int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d) * separated by ";". The "id" is in decimal, and must match one of * the "id"s for this resource. */ -static int parse_line(char *line, struct rdt_resource *r) +static int parse_line(char *line, struct rdt_resource *r, + struct rdtgroup *rdtgrp) { + struct rdt_cbm_parse_data data; char *dom = NULL, *id; struct rdt_domain *d; unsigned long dom_id; @@ -167,15 +212,32 @@ next: dom = strim(dom); list_for_each_entry(d, &r->domains, list) { if (d->id == dom_id) { - if (r->parse_ctrlval(dom, r, d)) + data.buf = dom; + data.rdtgrp = rdtgrp; + if (r->parse_ctrlval(&data, r, d)) return -EINVAL; + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + /* + * In pseudo-locking setup mode and just + * parsed a valid CBM that should be + * pseudo-locked. Only one locked region per + * resource group and domain so just do + * the required initialization for single + * region and return. + */ + rdtgrp->plr->r = r; + rdtgrp->plr->d = d; + rdtgrp->plr->cbm = d->new_ctrl; + d->plr = rdtgrp->plr; + return 0; + } goto next; } } return -EINVAL; } -static int update_domains(struct rdt_resource *r, int closid) +int update_domains(struct rdt_resource *r, int closid) { struct msr_param msr_param; cpumask_var_t cpu_mask; @@ -220,13 +282,14 @@ done: return 0; } -static int rdtgroup_parse_resource(char *resname, char *tok, int closid) +static int rdtgroup_parse_resource(char *resname, char *tok, + struct rdtgroup *rdtgrp) { struct rdt_resource *r; for_each_alloc_enabled_rdt_resource(r) { - if (!strcmp(resname, r->name) && closid < r->num_closid) - return parse_line(tok, r); + if (!strcmp(resname, r->name) && rdtgrp->closid < r->num_closid) + return parse_line(tok, r, rdtgrp); } rdt_last_cmd_printf("unknown/unsupported resource name '%s'\n", resname); return -EINVAL; @@ -239,7 +302,7 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, struct rdt_domain *dom; struct rdt_resource *r; char *tok, *resname; - int closid, ret = 0; + int ret = 0; /* Valid input requires a trailing newline */ if (nbytes == 0 || buf[nbytes - 1] != '\n') @@ -253,7 +316,15 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, } rdt_last_cmd_clear(); - closid = rdtgrp->closid; + /* + * No changes to pseudo-locked region allowed. It has to be removed + * and re-created instead. + */ + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + ret = -EINVAL; + rdt_last_cmd_puts("resource group is pseudo-locked\n"); + goto out; + } for_each_alloc_enabled_rdt_resource(r) { list_for_each_entry(dom, &r->domains, list) @@ -272,17 +343,27 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, ret = -EINVAL; goto out; } - ret = rdtgroup_parse_resource(resname, tok, closid); + ret = rdtgroup_parse_resource(resname, tok, rdtgrp); if (ret) goto out; } for_each_alloc_enabled_rdt_resource(r) { - ret = update_domains(r, closid); + ret = update_domains(r, rdtgrp->closid); if (ret) goto out; } + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + /* + * If pseudo-locking fails we keep the resource group in + * mode RDT_MODE_PSEUDO_LOCKSETUP with its class of service + * active and updated for just the domain the pseudo-locked + * region was requested for. + */ + ret = rdtgroup_pseudo_lock_create(rdtgrp); + } + out: rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; @@ -318,10 +399,18 @@ int rdtgroup_schemata_show(struct kernfs_open_file *of, rdtgrp = rdtgroup_kn_lock_live(of->kn); if (rdtgrp) { - closid = rdtgrp->closid; - for_each_alloc_enabled_rdt_resource(r) { - if (closid < r->num_closid) - show_doms(s, r, closid); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + for_each_alloc_enabled_rdt_resource(r) + seq_printf(s, "%s:uninitialized\n", r->name); + } else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + seq_printf(s, "%s:%d=%x\n", rdtgrp->plr->r->name, + rdtgrp->plr->d->id, rdtgrp->plr->cbm); + } else { + closid = rdtgrp->closid; + for_each_alloc_enabled_rdt_resource(r) { + if (closid < r->num_closid) + show_doms(s, r, closid); + } } } else { ret = -ENOENT; diff --git a/arch/x86/kernel/cpu/intel_rdt_pseudo_lock.c b/arch/x86/kernel/cpu/intel_rdt_pseudo_lock.c new file mode 100644 index 000000000000..40f3903ae5d9 --- /dev/null +++ b/arch/x86/kernel/cpu/intel_rdt_pseudo_lock.c @@ -0,0 +1,1522 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Resource Director Technology (RDT) + * + * Pseudo-locking support built on top of Cache Allocation Technology (CAT) + * + * Copyright (C) 2018 Intel Corporation + * + * Author: Reinette Chatre <reinette.chatre@intel.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cacheinfo.h> +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/debugfs.h> +#include <linux/kthread.h> +#include <linux/mman.h> +#include <linux/pm_qos.h> +#include <linux/slab.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/intel-family.h> +#include <asm/intel_rdt_sched.h> +#include <asm/perf_event.h> + +#include "intel_rdt.h" + +#define CREATE_TRACE_POINTS +#include "intel_rdt_pseudo_lock_event.h" + +/* + * MSR_MISC_FEATURE_CONTROL register enables the modification of hardware + * prefetcher state. Details about this register can be found in the MSR + * tables for specific platforms found in Intel's SDM. + */ +#define MSR_MISC_FEATURE_CONTROL 0x000001a4 + +/* + * The bits needed to disable hardware prefetching varies based on the + * platform. During initialization we will discover which bits to use. + */ +static u64 prefetch_disable_bits; + +/* + * Major number assigned to and shared by all devices exposing + * pseudo-locked regions. + */ +static unsigned int pseudo_lock_major; +static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0); +static struct class *pseudo_lock_class; + +/** + * get_prefetch_disable_bits - prefetch disable bits of supported platforms + * + * Capture the list of platforms that have been validated to support + * pseudo-locking. This includes testing to ensure pseudo-locked regions + * with low cache miss rates can be created under variety of load conditions + * as well as that these pseudo-locked regions can maintain their low cache + * miss rates under variety of load conditions for significant lengths of time. + * + * After a platform has been validated to support pseudo-locking its + * hardware prefetch disable bits are included here as they are documented + * in the SDM. + * + * When adding a platform here also add support for its cache events to + * measure_cycles_perf_fn() + * + * Return: + * If platform is supported, the bits to disable hardware prefetchers, 0 + * if platform is not supported. + */ +static u64 get_prefetch_disable_bits(void) +{ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || + boot_cpu_data.x86 != 6) + return 0; + + switch (boot_cpu_data.x86_model) { + case INTEL_FAM6_BROADWELL_X: + /* + * SDM defines bits of MSR_MISC_FEATURE_CONTROL register + * as: + * 0 L2 Hardware Prefetcher Disable (R/W) + * 1 L2 Adjacent Cache Line Prefetcher Disable (R/W) + * 2 DCU Hardware Prefetcher Disable (R/W) + * 3 DCU IP Prefetcher Disable (R/W) + * 63:4 Reserved + */ + return 0xF; + case INTEL_FAM6_ATOM_GOLDMONT: + case INTEL_FAM6_ATOM_GEMINI_LAKE: + /* + * SDM defines bits of MSR_MISC_FEATURE_CONTROL register + * as: + * 0 L2 Hardware Prefetcher Disable (R/W) + * 1 Reserved + * 2 DCU Hardware Prefetcher Disable (R/W) + * 63:3 Reserved + */ + return 0x5; + } + + return 0; +} + +/* + * Helper to write 64bit value to MSR without tracing. Used when + * use of the cache should be restricted and use of registers used + * for local variables avoided. + */ +static inline void pseudo_wrmsrl_notrace(unsigned int msr, u64 val) +{ + __wrmsr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32)); +} + +/** + * pseudo_lock_minor_get - Obtain available minor number + * @minor: Pointer to where new minor number will be stored + * + * A bitmask is used to track available minor numbers. Here the next free + * minor number is marked as unavailable and returned. + * + * Return: 0 on success, <0 on failure. + */ +static int pseudo_lock_minor_get(unsigned int *minor) +{ + unsigned long first_bit; + + first_bit = find_first_bit(&pseudo_lock_minor_avail, MINORBITS); + + if (first_bit == MINORBITS) + return -ENOSPC; + + __clear_bit(first_bit, &pseudo_lock_minor_avail); + *minor = first_bit; + + return 0; +} + +/** + * pseudo_lock_minor_release - Return minor number to available + * @minor: The minor number made available + */ +static void pseudo_lock_minor_release(unsigned int minor) +{ + __set_bit(minor, &pseudo_lock_minor_avail); +} + +/** + * region_find_by_minor - Locate a pseudo-lock region by inode minor number + * @minor: The minor number of the device representing pseudo-locked region + * + * When the character device is accessed we need to determine which + * pseudo-locked region it belongs to. This is done by matching the minor + * number of the device to the pseudo-locked region it belongs. + * + * Minor numbers are assigned at the time a pseudo-locked region is associated + * with a cache instance. + * + * Return: On success return pointer to resource group owning the pseudo-locked + * region, NULL on failure. + */ +static struct rdtgroup *region_find_by_minor(unsigned int minor) +{ + struct rdtgroup *rdtgrp, *rdtgrp_match = NULL; + + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (rdtgrp->plr && rdtgrp->plr->minor == minor) { + rdtgrp_match = rdtgrp; + break; + } + } + return rdtgrp_match; +} + +/** + * pseudo_lock_pm_req - A power management QoS request list entry + * @list: Entry within the @pm_reqs list for a pseudo-locked region + * @req: PM QoS request + */ +struct pseudo_lock_pm_req { + struct list_head list; + struct dev_pm_qos_request req; +}; + +static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) +{ + struct pseudo_lock_pm_req *pm_req, *next; + + list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) { + dev_pm_qos_remove_request(&pm_req->req); + list_del(&pm_req->list); + kfree(pm_req); + } +} + +/** + * pseudo_lock_cstates_constrain - Restrict cores from entering C6 + * + * To prevent the cache from being affected by power management entering + * C6 has to be avoided. This is accomplished by requesting a latency + * requirement lower than lowest C6 exit latency of all supported + * platforms as found in the cpuidle state tables in the intel_idle driver. + * At this time it is possible to do so with a single latency requirement + * for all supported platforms. + * + * Since Goldmont is supported, which is affected by X86_BUG_MONITOR, + * the ACPI latencies need to be considered while keeping in mind that C2 + * may be set to map to deeper sleep states. In this case the latency + * requirement needs to prevent entering C2 also. + */ +static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) +{ + struct pseudo_lock_pm_req *pm_req; + int cpu; + int ret; + + for_each_cpu(cpu, &plr->d->cpu_mask) { + pm_req = kzalloc(sizeof(*pm_req), GFP_KERNEL); + if (!pm_req) { + rdt_last_cmd_puts("fail allocating mem for PM QoS\n"); + ret = -ENOMEM; + goto out_err; + } + ret = dev_pm_qos_add_request(get_cpu_device(cpu), + &pm_req->req, + DEV_PM_QOS_RESUME_LATENCY, + 30); + if (ret < 0) { + rdt_last_cmd_printf("fail to add latency req cpu%d\n", + cpu); + kfree(pm_req); + ret = -1; + goto out_err; + } + list_add(&pm_req->list, &plr->pm_reqs); + } + + return 0; + +out_err: + pseudo_lock_cstates_relax(plr); + return ret; +} + +/** + * pseudo_lock_region_clear - Reset pseudo-lock region data + * @plr: pseudo-lock region + * + * All content of the pseudo-locked region is reset - any memory allocated + * freed. + * + * Return: void + */ +static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) +{ + plr->size = 0; + plr->line_size = 0; + kfree(plr->kmem); + plr->kmem = NULL; + plr->r = NULL; + if (plr->d) + plr->d->plr = NULL; + plr->d = NULL; + plr->cbm = 0; + plr->debugfs_dir = NULL; +} + +/** + * pseudo_lock_region_init - Initialize pseudo-lock region information + * @plr: pseudo-lock region + * + * Called after user provided a schemata to be pseudo-locked. From the + * schemata the &struct pseudo_lock_region is on entry already initialized + * with the resource, domain, and capacity bitmask. Here the information + * required for pseudo-locking is deduced from this data and &struct + * pseudo_lock_region initialized further. This information includes: + * - size in bytes of the region to be pseudo-locked + * - cache line size to know the stride with which data needs to be accessed + * to be pseudo-locked + * - a cpu associated with the cache instance on which the pseudo-locking + * flow can be executed + * + * Return: 0 on success, <0 on failure. Descriptive error will be written + * to last_cmd_status buffer. + */ +static int pseudo_lock_region_init(struct pseudo_lock_region *plr) +{ + struct cpu_cacheinfo *ci; + int ret; + int i; + + /* Pick the first cpu we find that is associated with the cache. */ + plr->cpu = cpumask_first(&plr->d->cpu_mask); + + if (!cpu_online(plr->cpu)) { + rdt_last_cmd_printf("cpu %u associated with cache not online\n", + plr->cpu); + ret = -ENODEV; + goto out_region; + } + + ci = get_cpu_cacheinfo(plr->cpu); + + plr->size = rdtgroup_cbm_to_size(plr->r, plr->d, plr->cbm); + + for (i = 0; i < ci->num_leaves; i++) { + if (ci->info_list[i].level == plr->r->cache_level) { + plr->line_size = ci->info_list[i].coherency_line_size; + return 0; + } + } + + ret = -1; + rdt_last_cmd_puts("unable to determine cache line size\n"); +out_region: + pseudo_lock_region_clear(plr); + return ret; +} + +/** + * pseudo_lock_init - Initialize a pseudo-lock region + * @rdtgrp: resource group to which new pseudo-locked region will belong + * + * A pseudo-locked region is associated with a resource group. When this + * association is created the pseudo-locked region is initialized. The + * details of the pseudo-locked region are not known at this time so only + * allocation is done and association established. + * + * Return: 0 on success, <0 on failure + */ +static int pseudo_lock_init(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr; + + plr = kzalloc(sizeof(*plr), GFP_KERNEL); + if (!plr) + return -ENOMEM; + + init_waitqueue_head(&plr->lock_thread_wq); + INIT_LIST_HEAD(&plr->pm_reqs); + rdtgrp->plr = plr; + return 0; +} + +/** + * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked + * @plr: pseudo-lock region + * + * Initialize the details required to set up the pseudo-locked region and + * allocate the contiguous memory that will be pseudo-locked to the cache. + * + * Return: 0 on success, <0 on failure. Descriptive error will be written + * to last_cmd_status buffer. + */ +static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr) +{ + int ret; + + ret = pseudo_lock_region_init(plr); + if (ret < 0) + return ret; + + /* + * We do not yet support contiguous regions larger than + * KMALLOC_MAX_SIZE. + */ + if (plr->size > KMALLOC_MAX_SIZE) { + rdt_last_cmd_puts("requested region exceeds maximum size\n"); + ret = -E2BIG; + goto out_region; + } + + plr->kmem = kzalloc(plr->size, GFP_KERNEL); + if (!plr->kmem) { + rdt_last_cmd_puts("unable to allocate memory\n"); + ret = -ENOMEM; + goto out_region; + } + + ret = 0; + goto out; +out_region: + pseudo_lock_region_clear(plr); +out: + return ret; +} + +/** + * pseudo_lock_free - Free a pseudo-locked region + * @rdtgrp: resource group to which pseudo-locked region belonged + * + * The pseudo-locked region's resources have already been released, or not + * yet created at this point. Now it can be freed and disassociated from the + * resource group. + * + * Return: void + */ +static void pseudo_lock_free(struct rdtgroup *rdtgrp) +{ + pseudo_lock_region_clear(rdtgrp->plr); + kfree(rdtgrp->plr); + rdtgrp->plr = NULL; +} + +/** + * pseudo_lock_fn - Load kernel memory into cache + * @_rdtgrp: resource group to which pseudo-lock region belongs + * + * This is the core pseudo-locking flow. + * + * First we ensure that the kernel memory cannot be found in the cache. + * Then, while taking care that there will be as little interference as + * possible, the memory to be loaded is accessed while core is running + * with class of service set to the bitmask of the pseudo-locked region. + * After this is complete no future CAT allocations will be allowed to + * overlap with this bitmask. + * + * Local register variables are utilized to ensure that the memory region + * to be locked is the only memory access made during the critical locking + * loop. + * + * Return: 0. Waiter on waitqueue will be woken on completion. + */ +static int pseudo_lock_fn(void *_rdtgrp) +{ + struct rdtgroup *rdtgrp = _rdtgrp; + struct pseudo_lock_region *plr = rdtgrp->plr; + u32 rmid_p, closid_p; + unsigned long i; +#ifdef CONFIG_KASAN + /* + * The registers used for local register variables are also used + * when KASAN is active. When KASAN is active we use a regular + * variable to ensure we always use a valid pointer, but the cost + * is that this variable will enter the cache through evicting the + * memory we are trying to lock into the cache. Thus expect lower + * pseudo-locking success rate when KASAN is active. + */ + unsigned int line_size; + unsigned int size; + void *mem_r; +#else + register unsigned int line_size asm("esi"); + register unsigned int size asm("edi"); +#ifdef CONFIG_X86_64 + register void *mem_r asm("rbx"); +#else + register void *mem_r asm("ebx"); +#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_KASAN */ + + /* + * Make sure none of the allocated memory is cached. If it is we + * will get a cache hit in below loop from outside of pseudo-locked + * region. + * wbinvd (as opposed to clflush/clflushopt) is required to + * increase likelihood that allocated cache portion will be filled + * with associated memory. + */ + native_wbinvd(); + + /* + * Always called with interrupts enabled. By disabling interrupts + * ensure that we will not be preempted during this critical section. + */ + local_irq_disable(); + + /* + * Call wrmsr and rdmsr as directly as possible to avoid tracing + * clobbering local register variables or affecting cache accesses. + * + * Disable the hardware prefetcher so that when the end of the memory + * being pseudo-locked is reached the hardware will not read beyond + * the buffer and evict pseudo-locked memory read earlier from the + * cache. + */ + __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + closid_p = this_cpu_read(pqr_state.cur_closid); + rmid_p = this_cpu_read(pqr_state.cur_rmid); + mem_r = plr->kmem; + size = plr->size; + line_size = plr->line_size; + /* + * Critical section begin: start by writing the closid associated + * with the capacity bitmask of the cache region being + * pseudo-locked followed by reading of kernel memory to load it + * into the cache. + */ + __wrmsr(IA32_PQR_ASSOC, rmid_p, rdtgrp->closid); + /* + * Cache was flushed earlier. Now access kernel memory to read it + * into cache region associated with just activated plr->closid. + * Loop over data twice: + * - In first loop the cache region is shared with the page walker + * as it populates the paging structure caches (including TLB). + * - In the second loop the paging structure caches are used and + * cache region is populated with the memory being referenced. + */ + for (i = 0; i < size; i += PAGE_SIZE) { + /* + * Add a barrier to prevent speculative execution of this + * loop reading beyond the end of the buffer. + */ + rmb(); + asm volatile("mov (%0,%1,1), %%eax\n\t" + : + : "r" (mem_r), "r" (i) + : "%eax", "memory"); + } + for (i = 0; i < size; i += line_size) { + /* + * Add a barrier to prevent speculative execution of this + * loop reading beyond the end of the buffer. + */ + rmb(); + asm volatile("mov (%0,%1,1), %%eax\n\t" + : + : "r" (mem_r), "r" (i) + : "%eax", "memory"); + } + /* + * Critical section end: restore closid with capacity bitmask that + * does not overlap with pseudo-locked region. + */ + __wrmsr(IA32_PQR_ASSOC, rmid_p, closid_p); + + /* Re-enable the hardware prefetcher(s) */ + wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + local_irq_enable(); + + plr->thread_done = 1; + wake_up_interruptible(&plr->lock_thread_wq); + return 0; +} + +/** + * rdtgroup_monitor_in_progress - Test if monitoring in progress + * @r: resource group being queried + * + * Return: 1 if monitor groups have been created for this resource + * group, 0 otherwise. + */ +static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp) +{ + return !list_empty(&rdtgrp->mon.crdtgrp_list); +} + +/** + * rdtgroup_locksetup_user_restrict - Restrict user access to group + * @rdtgrp: resource group needing access restricted + * + * A resource group used for cache pseudo-locking cannot have cpus or tasks + * assigned to it. This is communicated to the user by restricting access + * to all the files that can be used to make such changes. + * + * Permissions restored with rdtgroup_locksetup_user_restore() + * + * Return: 0 on success, <0 on failure. If a failure occurs during the + * restriction of access an attempt will be made to restore permissions but + * the state of the mode of these files will be uncertain when a failure + * occurs. + */ +static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp) +{ + int ret; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); + if (ret) + return ret; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); + if (ret) + goto err_tasks; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); + if (ret) + goto err_cpus; + + if (rdt_mon_capable) { + ret = rdtgroup_kn_mode_restrict(rdtgrp, "mon_groups"); + if (ret) + goto err_cpus_list; + } + + ret = 0; + goto out; + +err_cpus_list: + rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); +err_cpus: + rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); +err_tasks: + rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); +out: + return ret; +} + +/** + * rdtgroup_locksetup_user_restore - Restore user access to group + * @rdtgrp: resource group needing access restored + * + * Restore all file access previously removed using + * rdtgroup_locksetup_user_restrict() + * + * Return: 0 on success, <0 on failure. If a failure occurs during the + * restoration of access an attempt will be made to restrict permissions + * again but the state of the mode of these files will be uncertain when + * a failure occurs. + */ +static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp) +{ + int ret; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); + if (ret) + return ret; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); + if (ret) + goto err_tasks; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); + if (ret) + goto err_cpus; + + if (rdt_mon_capable) { + ret = rdtgroup_kn_mode_restore(rdtgrp, "mon_groups", 0777); + if (ret) + goto err_cpus_list; + } + + ret = 0; + goto out; + +err_cpus_list: + rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); +err_cpus: + rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); +err_tasks: + rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); +out: + return ret; +} + +/** + * rdtgroup_locksetup_enter - Resource group enters locksetup mode + * @rdtgrp: resource group requested to enter locksetup mode + * + * A resource group enters locksetup mode to reflect that it would be used + * to represent a pseudo-locked region and is in the process of being set + * up to do so. A resource group used for a pseudo-locked region would + * lose the closid associated with it so we cannot allow it to have any + * tasks or cpus assigned nor permit tasks or cpus to be assigned in the + * future. Monitoring of a pseudo-locked region is not allowed either. + * + * The above and more restrictions on a pseudo-locked region are checked + * for and enforced before the resource group enters the locksetup mode. + * + * Returns: 0 if the resource group successfully entered locksetup mode, <0 + * on failure. On failure the last_cmd_status buffer is updated with text to + * communicate details of failure to the user. + */ +int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) +{ + int ret; + + /* + * The default resource group can neither be removed nor lose the + * default closid associated with it. + */ + if (rdtgrp == &rdtgroup_default) { + rdt_last_cmd_puts("cannot pseudo-lock default group\n"); + return -EINVAL; + } + + /* + * Cache Pseudo-locking not supported when CDP is enabled. + * + * Some things to consider if you would like to enable this + * support (using L3 CDP as example): + * - When CDP is enabled two separate resources are exposed, + * L3DATA and L3CODE, but they are actually on the same cache. + * The implication for pseudo-locking is that if a + * pseudo-locked region is created on a domain of one + * resource (eg. L3CODE), then a pseudo-locked region cannot + * be created on that same domain of the other resource + * (eg. L3DATA). This is because the creation of a + * pseudo-locked region involves a call to wbinvd that will + * affect all cache allocations on particular domain. + * - Considering the previous, it may be possible to only + * expose one of the CDP resources to pseudo-locking and + * hide the other. For example, we could consider to only + * expose L3DATA and since the L3 cache is unified it is + * still possible to place instructions there are execute it. + * - If only one region is exposed to pseudo-locking we should + * still keep in mind that availability of a portion of cache + * for pseudo-locking should take into account both resources. + * Similarly, if a pseudo-locked region is created in one + * resource, the portion of cache used by it should be made + * unavailable to all future allocations from both resources. + */ + if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled || + rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled) { + rdt_last_cmd_puts("CDP enabled\n"); + return -EINVAL; + } + + /* + * Not knowing the bits to disable prefetching implies that this + * platform does not support Cache Pseudo-Locking. + */ + prefetch_disable_bits = get_prefetch_disable_bits(); + if (prefetch_disable_bits == 0) { + rdt_last_cmd_puts("pseudo-locking not supported\n"); + return -EINVAL; + } + + if (rdtgroup_monitor_in_progress(rdtgrp)) { + rdt_last_cmd_puts("monitoring in progress\n"); + return -EINVAL; + } + + if (rdtgroup_tasks_assigned(rdtgrp)) { + rdt_last_cmd_puts("tasks assigned to resource group\n"); + return -EINVAL; + } + + if (!cpumask_empty(&rdtgrp->cpu_mask)) { + rdt_last_cmd_puts("CPUs assigned to resource group\n"); + return -EINVAL; + } + + if (rdtgroup_locksetup_user_restrict(rdtgrp)) { + rdt_last_cmd_puts("unable to modify resctrl permissions\n"); + return -EIO; + } + + ret = pseudo_lock_init(rdtgrp); + if (ret) { + rdt_last_cmd_puts("unable to init pseudo-lock region\n"); + goto out_release; + } + + /* + * If this system is capable of monitoring a rmid would have been + * allocated when the control group was created. This is not needed + * anymore when this group would be used for pseudo-locking. This + * is safe to call on platforms not capable of monitoring. + */ + free_rmid(rdtgrp->mon.rmid); + + ret = 0; + goto out; + +out_release: + rdtgroup_locksetup_user_restore(rdtgrp); +out: + return ret; +} + +/** + * rdtgroup_locksetup_exit - resource group exist locksetup mode + * @rdtgrp: resource group + * + * When a resource group exits locksetup mode the earlier restrictions are + * lifted. + * + * Return: 0 on success, <0 on failure + */ +int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) +{ + int ret; + + if (rdt_mon_capable) { + ret = alloc_rmid(); + if (ret < 0) { + rdt_last_cmd_puts("out of RMIDs\n"); + return ret; + } + rdtgrp->mon.rmid = ret; + } + + ret = rdtgroup_locksetup_user_restore(rdtgrp); + if (ret) { + free_rmid(rdtgrp->mon.rmid); + return ret; + } + + pseudo_lock_free(rdtgrp); + return 0; +} + +/** + * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked + * @d: RDT domain + * @_cbm: CBM to test + * + * @d represents a cache instance and @_cbm a capacity bitmask that is + * considered for it. Determine if @_cbm overlaps with any existing + * pseudo-locked region on @d. + * + * Return: true if @_cbm overlaps with pseudo-locked region on @d, false + * otherwise. + */ +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_domain *d, u32 _cbm) +{ + unsigned long *cbm = (unsigned long *)&_cbm; + unsigned long *cbm_b; + unsigned int cbm_len; + + if (d->plr) { + cbm_len = d->plr->r->cache.cbm_len; + cbm_b = (unsigned long *)&d->plr->cbm; + if (bitmap_intersects(cbm, cbm_b, cbm_len)) + return true; + } + return false; +} + +/** + * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy + * @d: RDT domain under test + * + * The setup of a pseudo-locked region affects all cache instances within + * the hierarchy of the region. It is thus essential to know if any + * pseudo-locked regions exist within a cache hierarchy to prevent any + * attempts to create new pseudo-locked regions in the same hierarchy. + * + * Return: true if a pseudo-locked region exists in the hierarchy of @d or + * if it is not possible to test due to memory allocation issue, + * false otherwise. + */ +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) +{ + cpumask_var_t cpu_with_psl; + struct rdt_resource *r; + struct rdt_domain *d_i; + bool ret = false; + + if (!zalloc_cpumask_var(&cpu_with_psl, GFP_KERNEL)) + return true; + + /* + * First determine which cpus have pseudo-locked regions + * associated with them. + */ + for_each_alloc_enabled_rdt_resource(r) { + list_for_each_entry(d_i, &r->domains, list) { + if (d_i->plr) + cpumask_or(cpu_with_psl, cpu_with_psl, + &d_i->cpu_mask); + } + } + + /* + * Next test if new pseudo-locked region would intersect with + * existing region. + */ + if (cpumask_intersects(&d->cpu_mask, cpu_with_psl)) + ret = true; + + free_cpumask_var(cpu_with_psl); + return ret; +} + +/** + * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory + * @_plr: pseudo-lock region to measure + * + * There is no deterministic way to test if a memory region is cached. One + * way is to measure how long it takes to read the memory, the speed of + * access is a good way to learn how close to the cpu the data was. Even + * more, if the prefetcher is disabled and the memory is read at a stride + * of half the cache line, then a cache miss will be easy to spot since the + * read of the first half would be significantly slower than the read of + * the second half. + * + * Return: 0. Waiter on waitqueue will be woken on completion. + */ +static int measure_cycles_lat_fn(void *_plr) +{ + struct pseudo_lock_region *plr = _plr; + unsigned long i; + u64 start, end; +#ifdef CONFIG_KASAN + /* + * The registers used for local register variables are also used + * when KASAN is active. When KASAN is active we use a regular + * variable to ensure we always use a valid pointer to access memory. + * The cost is that accessing this pointer, which could be in + * cache, will be included in the measurement of memory read latency. + */ + void *mem_r; +#else +#ifdef CONFIG_X86_64 + register void *mem_r asm("rbx"); +#else + register void *mem_r asm("ebx"); +#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_KASAN */ + + local_irq_disable(); + /* + * The wrmsr call may be reordered with the assignment below it. + * Call wrmsr as directly as possible to avoid tracing clobbering + * local register variable used for memory pointer. + */ + __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + mem_r = plr->kmem; + /* + * Dummy execute of the time measurement to load the needed + * instructions into the L1 instruction cache. + */ + start = rdtsc_ordered(); + for (i = 0; i < plr->size; i += 32) { + start = rdtsc_ordered(); + asm volatile("mov (%0,%1,1), %%eax\n\t" + : + : "r" (mem_r), "r" (i) + : "%eax", "memory"); + end = rdtsc_ordered(); + trace_pseudo_lock_mem_latency((u32)(end - start)); + } + wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + local_irq_enable(); + plr->thread_done = 1; + wake_up_interruptible(&plr->lock_thread_wq); + return 0; +} + +static int measure_cycles_perf_fn(void *_plr) +{ + unsigned long long l3_hits = 0, l3_miss = 0; + u64 l3_hit_bits = 0, l3_miss_bits = 0; + struct pseudo_lock_region *plr = _plr; + unsigned long long l2_hits, l2_miss; + u64 l2_hit_bits, l2_miss_bits; + unsigned long i; +#ifdef CONFIG_KASAN + /* + * The registers used for local register variables are also used + * when KASAN is active. When KASAN is active we use regular variables + * at the cost of including cache access latency to these variables + * in the measurements. + */ + unsigned int line_size; + unsigned int size; + void *mem_r; +#else + register unsigned int line_size asm("esi"); + register unsigned int size asm("edi"); +#ifdef CONFIG_X86_64 + register void *mem_r asm("rbx"); +#else + register void *mem_r asm("ebx"); +#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_KASAN */ + + /* + * Non-architectural event for the Goldmont Microarchitecture + * from Intel x86 Architecture Software Developer Manual (SDM): + * MEM_LOAD_UOPS_RETIRED D1H (event number) + * Umask values: + * L1_HIT 01H + * L2_HIT 02H + * L1_MISS 08H + * L2_MISS 10H + * + * On Broadwell Microarchitecture the MEM_LOAD_UOPS_RETIRED event + * has two "no fix" errata associated with it: BDM35 and BDM100. On + * this platform we use the following events instead: + * L2_RQSTS 24H (Documented in https://download.01.org/perfmon/BDW/) + * REFERENCES FFH + * MISS 3FH + * LONGEST_LAT_CACHE 2EH (Documented in SDM) + * REFERENCE 4FH + * MISS 41H + */ + + /* + * Start by setting flags for IA32_PERFEVTSELx: + * OS (Operating system mode) 0x2 + * INT (APIC interrupt enable) 0x10 + * EN (Enable counter) 0x40 + * + * Then add the Umask value and event number to select performance + * event. + */ + + switch (boot_cpu_data.x86_model) { + case INTEL_FAM6_ATOM_GOLDMONT: + case INTEL_FAM6_ATOM_GEMINI_LAKE: + l2_hit_bits = (0x52ULL << 16) | (0x2 << 8) | 0xd1; + l2_miss_bits = (0x52ULL << 16) | (0x10 << 8) | 0xd1; + break; + case INTEL_FAM6_BROADWELL_X: + /* On BDW the l2_hit_bits count references, not hits */ + l2_hit_bits = (0x52ULL << 16) | (0xff << 8) | 0x24; + l2_miss_bits = (0x52ULL << 16) | (0x3f << 8) | 0x24; + /* On BDW the l3_hit_bits count references, not hits */ + l3_hit_bits = (0x52ULL << 16) | (0x4f << 8) | 0x2e; + l3_miss_bits = (0x52ULL << 16) | (0x41 << 8) | 0x2e; + break; + default: + goto out; + } + + local_irq_disable(); + /* + * Call wrmsr direcly to avoid the local register variables from + * being overwritten due to reordering of their assignment with + * the wrmsr calls. + */ + __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); + /* Disable events and reset counters */ + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 1, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_PERFCTR0, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_PERFCTR0 + 1, 0x0); + if (l3_hit_bits > 0) { + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 2, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 3, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_PERFCTR0 + 2, 0x0); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_PERFCTR0 + 3, 0x0); + } + /* Set and enable the L2 counters */ + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0, l2_hit_bits); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 1, l2_miss_bits); + if (l3_hit_bits > 0) { + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 2, + l3_hit_bits); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 3, + l3_miss_bits); + } + mem_r = plr->kmem; + size = plr->size; + line_size = plr->line_size; + for (i = 0; i < size; i += line_size) { + asm volatile("mov (%0,%1,1), %%eax\n\t" + : + : "r" (mem_r), "r" (i) + : "%eax", "memory"); + } + /* + * Call wrmsr directly (no tracing) to not influence + * the cache access counters as they are disabled. + */ + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0, + l2_hit_bits & ~(0x40ULL << 16)); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 1, + l2_miss_bits & ~(0x40ULL << 16)); + if (l3_hit_bits > 0) { + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 2, + l3_hit_bits & ~(0x40ULL << 16)); + pseudo_wrmsrl_notrace(MSR_ARCH_PERFMON_EVENTSEL0 + 3, + l3_miss_bits & ~(0x40ULL << 16)); + } + l2_hits = native_read_pmc(0); + l2_miss = native_read_pmc(1); + if (l3_hit_bits > 0) { + l3_hits = native_read_pmc(2); + l3_miss = native_read_pmc(3); + } + wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + local_irq_enable(); + /* + * On BDW we count references and misses, need to adjust. Sometimes + * the "hits" counter is a bit more than the references, for + * example, x references but x + 1 hits. To not report invalid + * hit values in this case we treat that as misses eaqual to + * references. + */ + if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) + l2_hits -= (l2_miss > l2_hits ? l2_hits : l2_miss); + trace_pseudo_lock_l2(l2_hits, l2_miss); + if (l3_hit_bits > 0) { + if (boot_cpu_data.x86_model == INTEL_FAM6_BROADWELL_X) + l3_hits -= (l3_miss > l3_hits ? l3_hits : l3_miss); + trace_pseudo_lock_l3(l3_hits, l3_miss); + } + +out: + plr->thread_done = 1; + wake_up_interruptible(&plr->lock_thread_wq); + return 0; +} + +/** + * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region + * + * The measurement of latency to access a pseudo-locked region should be + * done from a cpu that is associated with that pseudo-locked region. + * Determine which cpu is associated with this region and start a thread on + * that cpu to perform the measurement, wait for that thread to complete. + * + * Return: 0 on success, <0 on failure + */ +static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + struct task_struct *thread; + unsigned int cpu; + int ret = -1; + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + if (rdtgrp->flags & RDT_DELETED) { + ret = -ENODEV; + goto out; + } + + plr->thread_done = 0; + cpu = cpumask_first(&plr->d->cpu_mask); + if (!cpu_online(cpu)) { + ret = -ENODEV; + goto out; + } + + if (sel == 1) + thread = kthread_create_on_node(measure_cycles_lat_fn, plr, + cpu_to_node(cpu), + "pseudo_lock_measure/%u", + cpu); + else if (sel == 2) + thread = kthread_create_on_node(measure_cycles_perf_fn, plr, + cpu_to_node(cpu), + "pseudo_lock_measure/%u", + cpu); + else + goto out; + + if (IS_ERR(thread)) { + ret = PTR_ERR(thread); + goto out; + } + kthread_bind(thread, cpu); + wake_up_process(thread); + + ret = wait_event_interruptible(plr->lock_thread_wq, + plr->thread_done == 1); + if (ret < 0) + goto out; + + ret = 0; + +out: + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + return ret; +} + +static ssize_t pseudo_lock_measure_trigger(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct rdtgroup *rdtgrp = file->private_data; + size_t buf_size; + char buf[32]; + int ret; + int sel; + + buf_size = min(count, (sizeof(buf) - 1)); + if (copy_from_user(buf, user_buf, buf_size)) + return -EFAULT; + + buf[buf_size] = '\0'; + ret = kstrtoint(buf, 10, &sel); + if (ret == 0) { + if (sel != 1) + return -EINVAL; + ret = debugfs_file_get(file->f_path.dentry); + if (ret) + return ret; + ret = pseudo_lock_measure_cycles(rdtgrp, sel); + if (ret == 0) + ret = count; + debugfs_file_put(file->f_path.dentry); + } + + return ret; +} + +static const struct file_operations pseudo_measure_fops = { + .write = pseudo_lock_measure_trigger, + .open = simple_open, + .llseek = default_llseek, +}; + +/** + * rdtgroup_pseudo_lock_create - Create a pseudo-locked region + * @rdtgrp: resource group to which pseudo-lock region belongs + * + * Called when a resource group in the pseudo-locksetup mode receives a + * valid schemata that should be pseudo-locked. Since the resource group is + * in pseudo-locksetup mode the &struct pseudo_lock_region has already been + * allocated and initialized with the essential information. If a failure + * occurs the resource group remains in the pseudo-locksetup mode with the + * &struct pseudo_lock_region associated with it, but cleared from all + * information and ready for the user to re-attempt pseudo-locking by + * writing the schemata again. + * + * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0 + * on failure. Descriptive error will be written to last_cmd_status buffer. + */ +int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + struct task_struct *thread; + unsigned int new_minor; + struct device *dev; + int ret; + + ret = pseudo_lock_region_alloc(plr); + if (ret < 0) + return ret; + + ret = pseudo_lock_cstates_constrain(plr); + if (ret < 0) { + ret = -EINVAL; + goto out_region; + } + + plr->thread_done = 0; + + thread = kthread_create_on_node(pseudo_lock_fn, rdtgrp, + cpu_to_node(plr->cpu), + "pseudo_lock/%u", plr->cpu); + if (IS_ERR(thread)) { + ret = PTR_ERR(thread); + rdt_last_cmd_printf("locking thread returned error %d\n", ret); + goto out_cstates; + } + + kthread_bind(thread, plr->cpu); + wake_up_process(thread); + + ret = wait_event_interruptible(plr->lock_thread_wq, + plr->thread_done == 1); + if (ret < 0) { + /* + * If the thread does not get on the CPU for whatever + * reason and the process which sets up the region is + * interrupted then this will leave the thread in runnable + * state and once it gets on the CPU it will derefence + * the cleared, but not freed, plr struct resulting in an + * empty pseudo-locking loop. + */ + rdt_last_cmd_puts("locking thread interrupted\n"); + goto out_cstates; + } + + ret = pseudo_lock_minor_get(&new_minor); + if (ret < 0) { + rdt_last_cmd_puts("unable to obtain a new minor number\n"); + goto out_cstates; + } + + /* + * Unlock access but do not release the reference. The + * pseudo-locked region will still be here on return. + * + * The mutex has to be released temporarily to avoid a potential + * deadlock with the mm->mmap_sem semaphore which is obtained in + * the device_create() and debugfs_create_dir() callpath below + * as well as before the mmap() callback is called. + */ + mutex_unlock(&rdtgroup_mutex); + + if (!IS_ERR_OR_NULL(debugfs_resctrl)) { + plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name, + debugfs_resctrl); + if (!IS_ERR_OR_NULL(plr->debugfs_dir)) + debugfs_create_file("pseudo_lock_measure", 0200, + plr->debugfs_dir, rdtgrp, + &pseudo_measure_fops); + } + + dev = device_create(pseudo_lock_class, NULL, + MKDEV(pseudo_lock_major, new_minor), + rdtgrp, "%s", rdtgrp->kn->name); + + mutex_lock(&rdtgroup_mutex); + + if (IS_ERR(dev)) { + ret = PTR_ERR(dev); + rdt_last_cmd_printf("failed to create character device: %d\n", + ret); + goto out_debugfs; + } + + /* We released the mutex - check if group was removed while we did so */ + if (rdtgrp->flags & RDT_DELETED) { + ret = -ENODEV; + goto out_device; + } + + plr->minor = new_minor; + + rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED; + closid_free(rdtgrp->closid); + rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0444); + rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0444); + + ret = 0; + goto out; + +out_device: + device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, new_minor)); +out_debugfs: + debugfs_remove_recursive(plr->debugfs_dir); + pseudo_lock_minor_release(new_minor); +out_cstates: + pseudo_lock_cstates_relax(plr); +out_region: + pseudo_lock_region_clear(plr); +out: + return ret; +} + +/** + * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region + * @rdtgrp: resource group to which the pseudo-locked region belongs + * + * The removal of a pseudo-locked region can be initiated when the resource + * group is removed from user space via a "rmdir" from userspace or the + * unmount of the resctrl filesystem. On removal the resource group does + * not go back to pseudo-locksetup mode before it is removed, instead it is + * removed directly. There is thus assymmetry with the creation where the + * &struct pseudo_lock_region is removed here while it was not created in + * rdtgroup_pseudo_lock_create(). + * + * Return: void + */ +void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + /* + * Default group cannot be a pseudo-locked region so we can + * free closid here. + */ + closid_free(rdtgrp->closid); + goto free; + } + + pseudo_lock_cstates_relax(plr); + debugfs_remove_recursive(rdtgrp->plr->debugfs_dir); + device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor)); + pseudo_lock_minor_release(plr->minor); + +free: + pseudo_lock_free(rdtgrp); +} + +static int pseudo_lock_dev_open(struct inode *inode, struct file *filp) +{ + struct rdtgroup *rdtgrp; + + mutex_lock(&rdtgroup_mutex); + + rdtgrp = region_find_by_minor(iminor(inode)); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + + filp->private_data = rdtgrp; + atomic_inc(&rdtgrp->waitcount); + /* Perform a non-seekable open - llseek is not supported */ + filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); + + mutex_unlock(&rdtgroup_mutex); + + return 0; +} + +static int pseudo_lock_dev_release(struct inode *inode, struct file *filp) +{ + struct rdtgroup *rdtgrp; + + mutex_lock(&rdtgroup_mutex); + rdtgrp = filp->private_data; + WARN_ON(!rdtgrp); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + filp->private_data = NULL; + atomic_dec(&rdtgrp->waitcount); + mutex_unlock(&rdtgroup_mutex); + return 0; +} + +static int pseudo_lock_dev_mremap(struct vm_area_struct *area) +{ + /* Not supported */ + return -EINVAL; +} + +static const struct vm_operations_struct pseudo_mmap_ops = { + .mremap = pseudo_lock_dev_mremap, +}; + +static int pseudo_lock_dev_mmap(struct file *filp, struct vm_area_struct *vma) +{ + unsigned long vsize = vma->vm_end - vma->vm_start; + unsigned long off = vma->vm_pgoff << PAGE_SHIFT; + struct pseudo_lock_region *plr; + struct rdtgroup *rdtgrp; + unsigned long physical; + unsigned long psize; + + mutex_lock(&rdtgroup_mutex); + + rdtgrp = filp->private_data; + WARN_ON(!rdtgrp); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + + plr = rdtgrp->plr; + + /* + * Task is required to run with affinity to the cpus associated + * with the pseudo-locked region. If this is not the case the task + * may be scheduled elsewhere and invalidate entries in the + * pseudo-locked region. + */ + if (!cpumask_subset(¤t->cpus_allowed, &plr->d->cpu_mask)) { + mutex_unlock(&rdtgroup_mutex); + return -EINVAL; + } + + physical = __pa(plr->kmem) >> PAGE_SHIFT; + psize = plr->size - off; + + if (off > plr->size) { + mutex_unlock(&rdtgroup_mutex); + return -ENOSPC; + } + + /* + * Ensure changes are carried directly to the memory being mapped, + * do not allow copy-on-write mapping. + */ + if (!(vma->vm_flags & VM_SHARED)) { + mutex_unlock(&rdtgroup_mutex); + return -EINVAL; + } + + if (vsize > psize) { + mutex_unlock(&rdtgroup_mutex); + return -ENOSPC; + } + + memset(plr->kmem + off, 0, vsize); + + if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff, + vsize, vma->vm_page_prot)) { + mutex_unlock(&rdtgroup_mutex); + return -EAGAIN; + } + vma->vm_ops = &pseudo_mmap_ops; + mutex_unlock(&rdtgroup_mutex); + return 0; +} + +static const struct file_operations pseudo_lock_dev_fops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .read = NULL, + .write = NULL, + .open = pseudo_lock_dev_open, + .release = pseudo_lock_dev_release, + .mmap = pseudo_lock_dev_mmap, +}; + +static char *pseudo_lock_devnode(struct device *dev, umode_t *mode) +{ + struct rdtgroup *rdtgrp; + + rdtgrp = dev_get_drvdata(dev); + if (mode) + *mode = 0600; + return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name); +} + +int rdt_pseudo_lock_init(void) +{ + int ret; + + ret = register_chrdev(0, "pseudo_lock", &pseudo_lock_dev_fops); + if (ret < 0) + return ret; + + pseudo_lock_major = ret; + + pseudo_lock_class = class_create(THIS_MODULE, "pseudo_lock"); + if (IS_ERR(pseudo_lock_class)) { + ret = PTR_ERR(pseudo_lock_class); + unregister_chrdev(pseudo_lock_major, "pseudo_lock"); + return ret; + } + + pseudo_lock_class->devnode = pseudo_lock_devnode; + return 0; +} + +void rdt_pseudo_lock_release(void) +{ + class_destroy(pseudo_lock_class); + pseudo_lock_class = NULL; + unregister_chrdev(pseudo_lock_major, "pseudo_lock"); + pseudo_lock_major = 0; +} diff --git a/arch/x86/kernel/cpu/intel_rdt_pseudo_lock_event.h b/arch/x86/kernel/cpu/intel_rdt_pseudo_lock_event.h new file mode 100644 index 000000000000..2c041e6d9f05 --- /dev/null +++ b/arch/x86/kernel/cpu/intel_rdt_pseudo_lock_event.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM resctrl + +#if !defined(_TRACE_PSEUDO_LOCK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PSEUDO_LOCK_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(pseudo_lock_mem_latency, + TP_PROTO(u32 latency), + TP_ARGS(latency), + TP_STRUCT__entry(__field(u32, latency)), + TP_fast_assign(__entry->latency = latency), + TP_printk("latency=%u", __entry->latency) + ); + +TRACE_EVENT(pseudo_lock_l2, + TP_PROTO(u64 l2_hits, u64 l2_miss), + TP_ARGS(l2_hits, l2_miss), + TP_STRUCT__entry(__field(u64, l2_hits) + __field(u64, l2_miss)), + TP_fast_assign(__entry->l2_hits = l2_hits; + __entry->l2_miss = l2_miss;), + TP_printk("hits=%llu miss=%llu", + __entry->l2_hits, __entry->l2_miss)); + +TRACE_EVENT(pseudo_lock_l3, + TP_PROTO(u64 l3_hits, u64 l3_miss), + TP_ARGS(l3_hits, l3_miss), + TP_STRUCT__entry(__field(u64, l3_hits) + __field(u64, l3_miss)), + TP_fast_assign(__entry->l3_hits = l3_hits; + __entry->l3_miss = l3_miss;), + TP_printk("hits=%llu miss=%llu", + __entry->l3_hits, __entry->l3_miss)); + +#endif /* _TRACE_PSEUDO_LOCK_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE intel_rdt_pseudo_lock_event +#include <trace/define_trace.h> diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c index 749856a2e736..d6d7ea7349d0 100644 --- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c +++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c @@ -20,7 +20,9 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <linux/cacheinfo.h> #include <linux/cpu.h> +#include <linux/debugfs.h> #include <linux/fs.h> #include <linux/sysfs.h> #include <linux/kernfs.h> @@ -55,6 +57,8 @@ static struct kernfs_node *kn_mondata; static struct seq_buf last_cmd_status; static char last_cmd_status_buf[512]; +struct dentry *debugfs_resctrl; + void rdt_last_cmd_clear(void) { lockdep_assert_held(&rdtgroup_mutex); @@ -121,11 +125,65 @@ static int closid_alloc(void) return closid; } -static void closid_free(int closid) +void closid_free(int closid) { closid_free_map |= 1 << closid; } +/** + * closid_allocated - test if provided closid is in use + * @closid: closid to be tested + * + * Return: true if @closid is currently associated with a resource group, + * false if @closid is free + */ +static bool closid_allocated(unsigned int closid) +{ + return (closid_free_map & (1 << closid)) == 0; +} + +/** + * rdtgroup_mode_by_closid - Return mode of resource group with closid + * @closid: closid if the resource group + * + * Each resource group is associated with a @closid. Here the mode + * of a resource group can be queried by searching for it using its closid. + * + * Return: mode as &enum rdtgrp_mode of resource group with closid @closid + */ +enum rdtgrp_mode rdtgroup_mode_by_closid(int closid) +{ + struct rdtgroup *rdtgrp; + + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (rdtgrp->closid == closid) + return rdtgrp->mode; + } + + return RDT_NUM_MODES; +} + +static const char * const rdt_mode_str[] = { + [RDT_MODE_SHAREABLE] = "shareable", + [RDT_MODE_EXCLUSIVE] = "exclusive", + [RDT_MODE_PSEUDO_LOCKSETUP] = "pseudo-locksetup", + [RDT_MODE_PSEUDO_LOCKED] = "pseudo-locked", +}; + +/** + * rdtgroup_mode_str - Return the string representation of mode + * @mode: the resource group mode as &enum rdtgroup_mode + * + * Return: string representation of valid mode, "unknown" otherwise + */ +static const char *rdtgroup_mode_str(enum rdtgrp_mode mode) +{ + if (mode < RDT_MODE_SHAREABLE || mode >= RDT_NUM_MODES) + return "unknown"; + + return rdt_mode_str[mode]; +} + /* set uid and gid of rdtgroup dirs and files to that of the creator */ static int rdtgroup_kn_set_ugid(struct kernfs_node *kn) { @@ -207,8 +265,12 @@ static int rdtgroup_cpus_show(struct kernfs_open_file *of, rdtgrp = rdtgroup_kn_lock_live(of->kn); if (rdtgrp) { - seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n", - cpumask_pr_args(&rdtgrp->cpu_mask)); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) + seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n", + cpumask_pr_args(&rdtgrp->plr->d->cpu_mask)); + else + seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n", + cpumask_pr_args(&rdtgrp->cpu_mask)); } else { ret = -ENOENT; } @@ -394,6 +456,13 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, goto unlock; } + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = -EINVAL; + rdt_last_cmd_puts("pseudo-locking in progress\n"); + goto unlock; + } + if (is_cpu_list(of)) ret = cpulist_parse(buf, newmask); else @@ -509,6 +578,32 @@ static int __rdtgroup_move_task(struct task_struct *tsk, return ret; } +/** + * rdtgroup_tasks_assigned - Test if tasks have been assigned to resource group + * @r: Resource group + * + * Return: 1 if tasks have been assigned to @r, 0 otherwise + */ +int rdtgroup_tasks_assigned(struct rdtgroup *r) +{ + struct task_struct *p, *t; + int ret = 0; + + lockdep_assert_held(&rdtgroup_mutex); + + rcu_read_lock(); + for_each_process_thread(p, t) { + if ((r->type == RDTCTRL_GROUP && t->closid == r->closid) || + (r->type == RDTMON_GROUP && t->rmid == r->mon.rmid)) { + ret = 1; + break; + } + } + rcu_read_unlock(); + + return ret; +} + static int rdtgroup_task_write_permission(struct task_struct *task, struct kernfs_open_file *of) { @@ -570,13 +665,22 @@ static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of, if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) return -EINVAL; rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } rdt_last_cmd_clear(); - if (rdtgrp) - ret = rdtgroup_move_task(pid, rdtgrp, of); - else - ret = -ENOENT; + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = -EINVAL; + rdt_last_cmd_puts("pseudo-locking in progress\n"); + goto unlock; + } + ret = rdtgroup_move_task(pid, rdtgrp, of); + +unlock: rdtgroup_kn_unlock(of->kn); return ret ?: nbytes; @@ -662,6 +766,94 @@ static int rdt_shareable_bits_show(struct kernfs_open_file *of, return 0; } +/** + * rdt_bit_usage_show - Display current usage of resources + * + * A domain is a shared resource that can now be allocated differently. Here + * we display the current regions of the domain as an annotated bitmask. + * For each domain of this resource its allocation bitmask + * is annotated as below to indicate the current usage of the corresponding bit: + * 0 - currently unused + * X - currently available for sharing and used by software and hardware + * H - currently used by hardware only but available for software use + * S - currently used and shareable by software only + * E - currently used exclusively by one resource group + * P - currently pseudo-locked by one resource group + */ +static int rdt_bit_usage_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = of->kn->parent->priv; + u32 sw_shareable = 0, hw_shareable = 0; + u32 exclusive = 0, pseudo_locked = 0; + struct rdt_domain *dom; + int i, hwb, swb, excl, psl; + enum rdtgrp_mode mode; + bool sep = false; + u32 *ctrl; + + mutex_lock(&rdtgroup_mutex); + hw_shareable = r->cache.shareable_bits; + list_for_each_entry(dom, &r->domains, list) { + if (sep) + seq_putc(seq, ';'); + ctrl = dom->ctrl_val; + sw_shareable = 0; + exclusive = 0; + seq_printf(seq, "%d=", dom->id); + for (i = 0; i < r->num_closid; i++, ctrl++) { + if (!closid_allocated(i)) + continue; + mode = rdtgroup_mode_by_closid(i); + switch (mode) { + case RDT_MODE_SHAREABLE: + sw_shareable |= *ctrl; + break; + case RDT_MODE_EXCLUSIVE: + exclusive |= *ctrl; + break; + case RDT_MODE_PSEUDO_LOCKSETUP: + /* + * RDT_MODE_PSEUDO_LOCKSETUP is possible + * here but not included since the CBM + * associated with this CLOSID in this mode + * is not initialized and no task or cpu can be + * assigned this CLOSID. + */ + break; + case RDT_MODE_PSEUDO_LOCKED: + case RDT_NUM_MODES: + WARN(1, + "invalid mode for closid %d\n", i); + break; + } + } + for (i = r->cache.cbm_len - 1; i >= 0; i--) { + pseudo_locked = dom->plr ? dom->plr->cbm : 0; + hwb = test_bit(i, (unsigned long *)&hw_shareable); + swb = test_bit(i, (unsigned long *)&sw_shareable); + excl = test_bit(i, (unsigned long *)&exclusive); + psl = test_bit(i, (unsigned long *)&pseudo_locked); + if (hwb && swb) + seq_putc(seq, 'X'); + else if (hwb && !swb) + seq_putc(seq, 'H'); + else if (!hwb && swb) + seq_putc(seq, 'S'); + else if (excl) + seq_putc(seq, 'E'); + else if (psl) + seq_putc(seq, 'P'); + else /* Unused bits remain */ + seq_putc(seq, '0'); + } + sep = true; + } + seq_putc(seq, '\n'); + mutex_unlock(&rdtgroup_mutex); + return 0; +} + static int rdt_min_bw_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { @@ -740,6 +932,269 @@ static ssize_t max_threshold_occ_write(struct kernfs_open_file *of, return nbytes; } +/* + * rdtgroup_mode_show - Display mode of this resource group + */ +static int rdtgroup_mode_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + seq_printf(s, "%s\n", rdtgroup_mode_str(rdtgrp->mode)); + + rdtgroup_kn_unlock(of->kn); + return 0; +} + +/** + * rdtgroup_cbm_overlaps - Does CBM for intended closid overlap with other + * @r: Resource to which domain instance @d belongs. + * @d: The domain instance for which @closid is being tested. + * @cbm: Capacity bitmask being tested. + * @closid: Intended closid for @cbm. + * @exclusive: Only check if overlaps with exclusive resource groups + * + * Checks if provided @cbm intended to be used for @closid on domain + * @d overlaps with any other closids or other hardware usage associated + * with this domain. If @exclusive is true then only overlaps with + * resource groups in exclusive mode will be considered. If @exclusive + * is false then overlaps with any resource group or hardware entities + * will be considered. + * + * Return: false if CBM does not overlap, true if it does. + */ +bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d, + u32 _cbm, int closid, bool exclusive) +{ + unsigned long *cbm = (unsigned long *)&_cbm; + unsigned long *ctrl_b; + enum rdtgrp_mode mode; + u32 *ctrl; + int i; + + /* Check for any overlap with regions used by hardware directly */ + if (!exclusive) { + if (bitmap_intersects(cbm, + (unsigned long *)&r->cache.shareable_bits, + r->cache.cbm_len)) + return true; + } + + /* Check for overlap with other resource groups */ + ctrl = d->ctrl_val; + for (i = 0; i < r->num_closid; i++, ctrl++) { + ctrl_b = (unsigned long *)ctrl; + mode = rdtgroup_mode_by_closid(i); + if (closid_allocated(i) && i != closid && + mode != RDT_MODE_PSEUDO_LOCKSETUP) { + if (bitmap_intersects(cbm, ctrl_b, r->cache.cbm_len)) { + if (exclusive) { + if (mode == RDT_MODE_EXCLUSIVE) + return true; + continue; + } + return true; + } + } + } + + return false; +} + +/** + * rdtgroup_mode_test_exclusive - Test if this resource group can be exclusive + * + * An exclusive resource group implies that there should be no sharing of + * its allocated resources. At the time this group is considered to be + * exclusive this test can determine if its current schemata supports this + * setting by testing for overlap with all other resource groups. + * + * Return: true if resource group can be exclusive, false if there is overlap + * with allocations of other resource groups and thus this resource group + * cannot be exclusive. + */ +static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) +{ + int closid = rdtgrp->closid; + struct rdt_resource *r; + struct rdt_domain *d; + + for_each_alloc_enabled_rdt_resource(r) { + list_for_each_entry(d, &r->domains, list) { + if (rdtgroup_cbm_overlaps(r, d, d->ctrl_val[closid], + rdtgrp->closid, false)) + return false; + } + } + + return true; +} + +/** + * rdtgroup_mode_write - Modify the resource group's mode + * + */ +static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct rdtgroup *rdtgrp; + enum rdtgrp_mode mode; + int ret = 0; + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') + return -EINVAL; + buf[nbytes - 1] = '\0'; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + rdt_last_cmd_clear(); + + mode = rdtgrp->mode; + + if ((!strcmp(buf, "shareable") && mode == RDT_MODE_SHAREABLE) || + (!strcmp(buf, "exclusive") && mode == RDT_MODE_EXCLUSIVE) || + (!strcmp(buf, "pseudo-locksetup") && + mode == RDT_MODE_PSEUDO_LOCKSETUP) || + (!strcmp(buf, "pseudo-locked") && mode == RDT_MODE_PSEUDO_LOCKED)) + goto out; + + if (mode == RDT_MODE_PSEUDO_LOCKED) { + rdt_last_cmd_printf("cannot change pseudo-locked group\n"); + ret = -EINVAL; + goto out; + } + + if (!strcmp(buf, "shareable")) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = rdtgroup_locksetup_exit(rdtgrp); + if (ret) + goto out; + } + rdtgrp->mode = RDT_MODE_SHAREABLE; + } else if (!strcmp(buf, "exclusive")) { + if (!rdtgroup_mode_test_exclusive(rdtgrp)) { + rdt_last_cmd_printf("schemata overlaps\n"); + ret = -EINVAL; + goto out; + } + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = rdtgroup_locksetup_exit(rdtgrp); + if (ret) + goto out; + } + rdtgrp->mode = RDT_MODE_EXCLUSIVE; + } else if (!strcmp(buf, "pseudo-locksetup")) { + ret = rdtgroup_locksetup_enter(rdtgrp); + if (ret) + goto out; + rdtgrp->mode = RDT_MODE_PSEUDO_LOCKSETUP; + } else { + rdt_last_cmd_printf("unknown/unsupported mode\n"); + ret = -EINVAL; + } + +out: + rdtgroup_kn_unlock(of->kn); + return ret ?: nbytes; +} + +/** + * rdtgroup_cbm_to_size - Translate CBM to size in bytes + * @r: RDT resource to which @d belongs. + * @d: RDT domain instance. + * @cbm: bitmask for which the size should be computed. + * + * The bitmask provided associated with the RDT domain instance @d will be + * translated into how many bytes it represents. The size in bytes is + * computed by first dividing the total cache size by the CBM length to + * determine how many bytes each bit in the bitmask represents. The result + * is multiplied with the number of bits set in the bitmask. + */ +unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, + struct rdt_domain *d, u32 cbm) +{ + struct cpu_cacheinfo *ci; + unsigned int size = 0; + int num_b, i; + + num_b = bitmap_weight((unsigned long *)&cbm, r->cache.cbm_len); + ci = get_cpu_cacheinfo(cpumask_any(&d->cpu_mask)); + for (i = 0; i < ci->num_leaves; i++) { + if (ci->info_list[i].level == r->cache_level) { + size = ci->info_list[i].size / r->cache.cbm_len * num_b; + break; + } + } + + return size; +} + +/** + * rdtgroup_size_show - Display size in bytes of allocated regions + * + * The "size" file mirrors the layout of the "schemata" file, printing the + * size in bytes of each region instead of the capacity bitmask. + * + */ +static int rdtgroup_size_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + struct rdt_resource *r; + struct rdt_domain *d; + unsigned int size; + bool sep = false; + u32 cbm; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + seq_printf(s, "%*s:", max_name_width, rdtgrp->plr->r->name); + size = rdtgroup_cbm_to_size(rdtgrp->plr->r, + rdtgrp->plr->d, + rdtgrp->plr->cbm); + seq_printf(s, "%d=%u\n", rdtgrp->plr->d->id, size); + goto out; + } + + for_each_alloc_enabled_rdt_resource(r) { + seq_printf(s, "%*s:", max_name_width, r->name); + list_for_each_entry(d, &r->domains, list) { + if (sep) + seq_putc(s, ';'); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + size = 0; + } else { + cbm = d->ctrl_val[rdtgrp->closid]; + size = rdtgroup_cbm_to_size(r, d, cbm); + } + seq_printf(s, "%d=%u", d->id, size); + sep = true; + } + seq_putc(s, '\n'); + } + +out: + rdtgroup_kn_unlock(of->kn); + + return 0; +} + /* rdtgroup information files for one cache resource. */ static struct rftype res_common_files[] = { { @@ -792,6 +1247,13 @@ static struct rftype res_common_files[] = { .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE, }, { + .name = "bit_usage", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_bit_usage_show, + .fflags = RF_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { .name = "min_bandwidth", .mode = 0444, .kf_ops = &rdtgroup_kf_single_ops, @@ -853,6 +1315,22 @@ static struct rftype res_common_files[] = { .seq_show = rdtgroup_schemata_show, .fflags = RF_CTRL_BASE, }, + { + .name = "mode", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_mode_write, + .seq_show = rdtgroup_mode_show, + .fflags = RF_CTRL_BASE, + }, + { + .name = "size", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdtgroup_size_show, + .fflags = RF_CTRL_BASE, + }, + }; static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags) @@ -883,6 +1361,103 @@ error: return ret; } +/** + * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file + * @r: The resource group with which the file is associated. + * @name: Name of the file + * + * The permissions of named resctrl file, directory, or link are modified + * to not allow read, write, or execute by any user. + * + * WARNING: This function is intended to communicate to the user that the + * resctrl file has been locked down - that it is not relevant to the + * particular state the system finds itself in. It should not be relied + * on to protect from user access because after the file's permissions + * are restricted the user can still change the permissions using chmod + * from the command line. + * + * Return: 0 on success, <0 on failure. + */ +int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name) +{ + struct iattr iattr = {.ia_valid = ATTR_MODE,}; + struct kernfs_node *kn; + int ret = 0; + + kn = kernfs_find_and_get_ns(r->kn, name, NULL); + if (!kn) + return -ENOENT; + + switch (kernfs_type(kn)) { + case KERNFS_DIR: + iattr.ia_mode = S_IFDIR; + break; + case KERNFS_FILE: + iattr.ia_mode = S_IFREG; + break; + case KERNFS_LINK: + iattr.ia_mode = S_IFLNK; + break; + } + + ret = kernfs_setattr(kn, &iattr); + kernfs_put(kn); + return ret; +} + +/** + * rdtgroup_kn_mode_restore - Restore user access to named resctrl file + * @r: The resource group with which the file is associated. + * @name: Name of the file + * @mask: Mask of permissions that should be restored + * + * Restore the permissions of the named file. If @name is a directory the + * permissions of its parent will be used. + * + * Return: 0 on success, <0 on failure. + */ +int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, + umode_t mask) +{ + struct iattr iattr = {.ia_valid = ATTR_MODE,}; + struct kernfs_node *kn, *parent; + struct rftype *rfts, *rft; + int ret, len; + + rfts = res_common_files; + len = ARRAY_SIZE(res_common_files); + + for (rft = rfts; rft < rfts + len; rft++) { + if (!strcmp(rft->name, name)) + iattr.ia_mode = rft->mode & mask; + } + + kn = kernfs_find_and_get_ns(r->kn, name, NULL); + if (!kn) + return -ENOENT; + + switch (kernfs_type(kn)) { + case KERNFS_DIR: + parent = kernfs_get_parent(kn); + if (parent) { + iattr.ia_mode |= parent->mode; + kernfs_put(parent); + } + iattr.ia_mode |= S_IFDIR; + break; + case KERNFS_FILE: + iattr.ia_mode |= S_IFREG; + break; + case KERNFS_LINK: + iattr.ia_mode |= S_IFLNK; + break; + } + + ret = kernfs_setattr(kn, &iattr); + kernfs_put(kn); + return ret; +} + static int rdtgroup_mkdir_info_resdir(struct rdt_resource *r, char *name, unsigned long fflags) { @@ -1224,6 +1799,9 @@ void rdtgroup_kn_unlock(struct kernfs_node *kn) if (atomic_dec_and_test(&rdtgrp->waitcount) && (rdtgrp->flags & RDT_DELETED)) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) + rdtgroup_pseudo_lock_remove(rdtgrp); kernfs_unbreak_active_protection(kn); kernfs_put(rdtgrp->kn); kfree(rdtgrp); @@ -1289,10 +1867,16 @@ static struct dentry *rdt_mount(struct file_system_type *fs_type, rdtgroup_default.mon.mon_data_kn = kn_mondata; } + ret = rdt_pseudo_lock_init(); + if (ret) { + dentry = ERR_PTR(ret); + goto out_mondata; + } + dentry = kernfs_mount(fs_type, flags, rdt_root, RDTGROUP_SUPER_MAGIC, NULL); if (IS_ERR(dentry)) - goto out_mondata; + goto out_psl; if (rdt_alloc_capable) static_branch_enable_cpuslocked(&rdt_alloc_enable_key); @@ -1310,6 +1894,8 @@ static struct dentry *rdt_mount(struct file_system_type *fs_type, goto out; +out_psl: + rdt_pseudo_lock_release(); out_mondata: if (rdt_mon_capable) kernfs_remove(kn_mondata); @@ -1447,6 +2033,10 @@ static void rmdir_all_sub(void) if (rdtgrp == &rdtgroup_default) continue; + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) + rdtgroup_pseudo_lock_remove(rdtgrp); + /* * Give any CPUs back to the default group. We cannot copy * cpu_online_mask because a CPU might have executed the @@ -1483,6 +2073,8 @@ static void rdt_kill_sb(struct super_block *sb) reset_all_ctrls(r); cdp_disable_all(); rmdir_all_sub(); + rdt_pseudo_lock_release(); + rdtgroup_default.mode = RDT_MODE_SHAREABLE; static_branch_disable_cpuslocked(&rdt_alloc_enable_key); static_branch_disable_cpuslocked(&rdt_mon_enable_key); static_branch_disable_cpuslocked(&rdt_enable_key); @@ -1682,6 +2274,114 @@ out_destroy: return ret; } +/** + * cbm_ensure_valid - Enforce validity on provided CBM + * @_val: Candidate CBM + * @r: RDT resource to which the CBM belongs + * + * The provided CBM represents all cache portions available for use. This + * may be represented by a bitmap that does not consist of contiguous ones + * and thus be an invalid CBM. + * Here the provided CBM is forced to be a valid CBM by only considering + * the first set of contiguous bits as valid and clearing all bits. + * The intention here is to provide a valid default CBM with which a new + * resource group is initialized. The user can follow this with a + * modification to the CBM if the default does not satisfy the + * requirements. + */ +static void cbm_ensure_valid(u32 *_val, struct rdt_resource *r) +{ + /* + * Convert the u32 _val to an unsigned long required by all the bit + * operations within this function. No more than 32 bits of this + * converted value can be accessed because all bit operations are + * additionally provided with cbm_len that is initialized during + * hardware enumeration using five bits from the EAX register and + * thus never can exceed 32 bits. + */ + unsigned long *val = (unsigned long *)_val; + unsigned int cbm_len = r->cache.cbm_len; + unsigned long first_bit, zero_bit; + + if (*val == 0) + return; + + first_bit = find_first_bit(val, cbm_len); + zero_bit = find_next_zero_bit(val, cbm_len, first_bit); + + /* Clear any remaining bits to ensure contiguous region */ + bitmap_clear(val, zero_bit, cbm_len - zero_bit); +} + +/** + * rdtgroup_init_alloc - Initialize the new RDT group's allocations + * + * A new RDT group is being created on an allocation capable (CAT) + * supporting system. Set this group up to start off with all usable + * allocations. That is, all shareable and unused bits. + * + * All-zero CBM is invalid. If there are no more shareable bits available + * on any domain then the entire allocation will fail. + */ +static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp) +{ + u32 used_b = 0, unused_b = 0; + u32 closid = rdtgrp->closid; + struct rdt_resource *r; + enum rdtgrp_mode mode; + struct rdt_domain *d; + int i, ret; + u32 *ctrl; + + for_each_alloc_enabled_rdt_resource(r) { + list_for_each_entry(d, &r->domains, list) { + d->have_new_ctrl = false; + d->new_ctrl = r->cache.shareable_bits; + used_b = r->cache.shareable_bits; + ctrl = d->ctrl_val; + for (i = 0; i < r->num_closid; i++, ctrl++) { + if (closid_allocated(i) && i != closid) { + mode = rdtgroup_mode_by_closid(i); + if (mode == RDT_MODE_PSEUDO_LOCKSETUP) + break; + used_b |= *ctrl; + if (mode == RDT_MODE_SHAREABLE) + d->new_ctrl |= *ctrl; + } + } + if (d->plr && d->plr->cbm > 0) + used_b |= d->plr->cbm; + unused_b = used_b ^ (BIT_MASK(r->cache.cbm_len) - 1); + unused_b &= BIT_MASK(r->cache.cbm_len) - 1; + d->new_ctrl |= unused_b; + /* + * Force the initial CBM to be valid, user can + * modify the CBM based on system availability. + */ + cbm_ensure_valid(&d->new_ctrl, r); + if (bitmap_weight((unsigned long *) &d->new_ctrl, + r->cache.cbm_len) < + r->cache.min_cbm_bits) { + rdt_last_cmd_printf("no space on %s:%d\n", + r->name, d->id); + return -ENOSPC; + } + d->have_new_ctrl = true; + } + } + + for_each_alloc_enabled_rdt_resource(r) { + ret = update_domains(r, rdtgrp->closid); + if (ret < 0) { + rdt_last_cmd_puts("failed to initialize allocations\n"); + return ret; + } + rdtgrp->mode = RDT_MODE_SHAREABLE; + } + + return 0; +} + static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, struct kernfs_node *prgrp_kn, const char *name, umode_t mode, @@ -1700,6 +2400,14 @@ static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, goto out_unlock; } + if (rtype == RDTMON_GROUP && + (prdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + prdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) { + ret = -EINVAL; + rdt_last_cmd_puts("pseudo-locking in progress\n"); + goto out_unlock; + } + /* allocate the rdtgroup. */ rdtgrp = kzalloc(sizeof(*rdtgrp), GFP_KERNEL); if (!rdtgrp) { @@ -1840,6 +2548,10 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, ret = 0; rdtgrp->closid = closid; + ret = rdtgroup_init_alloc(rdtgrp); + if (ret < 0) + goto out_id_free; + list_add(&rdtgrp->rdtgroup_list, &rdt_all_groups); if (rdt_mon_capable) { @@ -1850,15 +2562,16 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, ret = mongroup_create_dir(kn, NULL, "mon_groups", NULL); if (ret) { rdt_last_cmd_puts("kernfs subdir error\n"); - goto out_id_free; + goto out_del_list; } } goto out_unlock; +out_del_list: + list_del(&rdtgrp->rdtgroup_list); out_id_free: closid_free(closid); - list_del(&rdtgrp->rdtgroup_list); out_common_fail: mkdir_rdt_prepare_clean(rdtgrp); out_unlock: @@ -1945,6 +2658,21 @@ static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp, return 0; } +static int rdtgroup_ctrl_remove(struct kernfs_node *kn, + struct rdtgroup *rdtgrp) +{ + rdtgrp->flags = RDT_DELETED; + list_del(&rdtgrp->rdtgroup_list); + + /* + * one extra hold on this, will drop when we kfree(rdtgrp) + * in rdtgroup_kn_unlock() + */ + kernfs_get(kn); + kernfs_remove(rdtgrp->kn); + return 0; +} + static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { @@ -1970,7 +2698,6 @@ static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask); update_closid_rmid(tmpmask, NULL); - rdtgrp->flags = RDT_DELETED; closid_free(rdtgrp->closid); free_rmid(rdtgrp->mon.rmid); @@ -1979,14 +2706,7 @@ static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, */ free_all_child_rdtgrp(rdtgrp); - list_del(&rdtgrp->rdtgroup_list); - - /* - * one extra hold on this, will drop when we kfree(rdtgrp) - * in rdtgroup_kn_unlock() - */ - kernfs_get(kn); - kernfs_remove(rdtgrp->kn); + rdtgroup_ctrl_remove(kn, rdtgrp); return 0; } @@ -2014,13 +2734,19 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) * If the rdtgroup is a mon group and parent directory * is a valid "mon_groups" directory, remove the mon group. */ - if (rdtgrp->type == RDTCTRL_GROUP && parent_kn == rdtgroup_default.kn) - ret = rdtgroup_rmdir_ctrl(kn, rdtgrp, tmpmask); - else if (rdtgrp->type == RDTMON_GROUP && - is_mon_groups(parent_kn, kn->name)) + if (rdtgrp->type == RDTCTRL_GROUP && parent_kn == rdtgroup_default.kn) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + ret = rdtgroup_ctrl_remove(kn, rdtgrp); + } else { + ret = rdtgroup_rmdir_ctrl(kn, rdtgrp, tmpmask); + } + } else if (rdtgrp->type == RDTMON_GROUP && + is_mon_groups(parent_kn, kn->name)) { ret = rdtgroup_rmdir_mon(kn, rdtgrp, tmpmask); - else + } else { ret = -EPERM; + } out: rdtgroup_kn_unlock(kn); @@ -2046,7 +2772,8 @@ static int __init rdtgroup_setup_root(void) int ret; rdt_root = kernfs_create_root(&rdtgroup_kf_syscall_ops, - KERNFS_ROOT_CREATE_DEACTIVATED, + KERNFS_ROOT_CREATE_DEACTIVATED | + KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, &rdtgroup_default); if (IS_ERR(rdt_root)) return PTR_ERR(rdt_root); @@ -2102,6 +2829,29 @@ int __init rdtgroup_init(void) if (ret) goto cleanup_mountpoint; + /* + * Adding the resctrl debugfs directory here may not be ideal since + * it would let the resctrl debugfs directory appear on the debugfs + * filesystem before the resctrl filesystem is mounted. + * It may also be ok since that would enable debugging of RDT before + * resctrl is mounted. + * The reason why the debugfs directory is created here and not in + * rdt_mount() is because rdt_mount() takes rdtgroup_mutex and + * during the debugfs directory creation also &sb->s_type->i_mutex_key + * (the lockdep class of inode->i_rwsem). Other filesystem + * interactions (eg. SyS_getdents) have the lock ordering: + * &sb->s_type->i_mutex_key --> &mm->mmap_sem + * During mmap(), called with &mm->mmap_sem, the rdtgroup_mutex + * is taken, thus creating dependency: + * &mm->mmap_sem --> rdtgroup_mutex for the latter that can cause + * issues considering the other two lock dependencies. + * By creating the debugfs directory here we avoid a dependency + * that may cause deadlock (even though file operations cannot + * occur until the filesystem is mounted, but I do not know how to + * tell lockdep that). + */ + debugfs_resctrl = debugfs_create_dir("resctrl", NULL); + return 0; cleanup_mountpoint: @@ -2111,3 +2861,11 @@ cleanup_root: return ret; } + +void __exit rdtgroup_exit(void) +{ + debugfs_remove_recursive(debugfs_resctrl); + unregister_filesystem(&rdt_fs_type); + sysfs_remove_mount_point(fs_kobj, "resctrl"); + kernfs_destroy_root(rdt_root); +} diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index c102ad51025e..4b767284b7f5 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -123,8 +123,8 @@ void mce_setup(struct mce *m) { memset(m, 0, sizeof(struct mce)); m->cpu = m->extcpu = smp_processor_id(); - /* We hope get_seconds stays lockless */ - m->time = get_seconds(); + /* need the internal __ version to avoid deadlocks */ + m->time = __ktime_get_real_seconds(); m->cpuvendor = boot_cpu_data.x86_vendor; m->cpuid = cpuid_eax(1); m->socketid = cpu_data(m->extcpu).phys_proc_id; @@ -1104,6 +1104,101 @@ static void mce_unmap_kpfn(unsigned long pfn) } #endif + +/* + * Cases where we avoid rendezvous handler timeout: + * 1) If this CPU is offline. + * + * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to + * skip those CPUs which remain looping in the 1st kernel - see + * crash_nmi_callback(). + * + * Note: there still is a small window between kexec-ing and the new, + * kdump kernel establishing a new #MC handler where a broadcasted MCE + * might not get handled properly. + */ +static bool __mc_check_crashing_cpu(int cpu) +{ + if (cpu_is_offline(cpu) || + (crashing_cpu != -1 && crashing_cpu != cpu)) { + u64 mcgstatus; + + mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); + if (mcgstatus & MCG_STATUS_RIPV) { + mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); + return true; + } + } + return false; +} + +static void __mc_scan_banks(struct mce *m, struct mce *final, + unsigned long *toclear, unsigned long *valid_banks, + int no_way_out, int *worst) +{ + struct mca_config *cfg = &mca_cfg; + int severity, i; + + for (i = 0; i < cfg->banks; i++) { + __clear_bit(i, toclear); + if (!test_bit(i, valid_banks)) + continue; + + if (!mce_banks[i].ctl) + continue; + + m->misc = 0; + m->addr = 0; + m->bank = i; + + m->status = mce_rdmsrl(msr_ops.status(i)); + if (!(m->status & MCI_STATUS_VAL)) + continue; + + /* + * Corrected or non-signaled errors are handled by + * machine_check_poll(). Leave them alone, unless this panics. + */ + if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && + !no_way_out) + continue; + + /* Set taint even when machine check was not enabled. */ + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); + + severity = mce_severity(m, cfg->tolerant, NULL, true); + + /* + * When machine check was for corrected/deferred handler don't + * touch, unless we're panicking. + */ + if ((severity == MCE_KEEP_SEVERITY || + severity == MCE_UCNA_SEVERITY) && !no_way_out) + continue; + + __set_bit(i, toclear); + + /* Machine check event was not enabled. Clear, but ignore. */ + if (severity == MCE_NO_SEVERITY) + continue; + + mce_read_aux(m, i); + + /* assuming valid severity level != 0 */ + m->severity = severity; + + mce_log(m); + + if (severity > *worst) { + *final = *m; + *worst = severity; + } + } + + /* mce_clear_state will clear *final, save locally for use later */ + *m = *final; +} + /* * The actual machine check handler. This only handles real * exceptions when something got corrupted coming in through int 18. @@ -1118,68 +1213,45 @@ static void mce_unmap_kpfn(unsigned long pfn) */ void do_machine_check(struct pt_regs *regs, long error_code) { + DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); + DECLARE_BITMAP(toclear, MAX_NR_BANKS); struct mca_config *cfg = &mca_cfg; + int cpu = smp_processor_id(); + char *msg = "Unknown"; struct mce m, *final; - int i; int worst = 0; - int severity; /* * Establish sequential order between the CPUs entering the machine * check handler. */ int order = -1; + /* * If no_way_out gets set, there is no safe way to recover from this * MCE. If mca_cfg.tolerant is cranked up, we'll try anyway. */ int no_way_out = 0; + /* * If kill_it gets set, there might be a way to recover from this * error. */ int kill_it = 0; - DECLARE_BITMAP(toclear, MAX_NR_BANKS); - DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); - char *msg = "Unknown"; /* * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES * on Intel. */ int lmce = 1; - int cpu = smp_processor_id(); - - /* - * Cases where we avoid rendezvous handler timeout: - * 1) If this CPU is offline. - * - * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to - * skip those CPUs which remain looping in the 1st kernel - see - * crash_nmi_callback(). - * - * Note: there still is a small window between kexec-ing and the new, - * kdump kernel establishing a new #MC handler where a broadcasted MCE - * might not get handled properly. - */ - if (cpu_is_offline(cpu) || - (crashing_cpu != -1 && crashing_cpu != cpu)) { - u64 mcgstatus; - mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); - if (mcgstatus & MCG_STATUS_RIPV) { - mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); - return; - } - } + if (__mc_check_crashing_cpu(cpu)) + return; ist_enter(regs); this_cpu_inc(mce_exception_count); - if (!cfg->banks) - goto out; - mce_gather_info(&m, regs); m.tsc = rdtsc(); @@ -1220,67 +1292,7 @@ void do_machine_check(struct pt_regs *regs, long error_code) order = mce_start(&no_way_out); } - for (i = 0; i < cfg->banks; i++) { - __clear_bit(i, toclear); - if (!test_bit(i, valid_banks)) - continue; - if (!mce_banks[i].ctl) - continue; - - m.misc = 0; - m.addr = 0; - m.bank = i; - - m.status = mce_rdmsrl(msr_ops.status(i)); - if ((m.status & MCI_STATUS_VAL) == 0) - continue; - - /* - * Non uncorrected or non signaled errors are handled by - * machine_check_poll. Leave them alone, unless this panics. - */ - if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && - !no_way_out) - continue; - - /* - * Set taint even when machine check was not enabled. - */ - add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); - - severity = mce_severity(&m, cfg->tolerant, NULL, true); - - /* - * When machine check was for corrected/deferred handler don't - * touch, unless we're panicing. - */ - if ((severity == MCE_KEEP_SEVERITY || - severity == MCE_UCNA_SEVERITY) && !no_way_out) - continue; - __set_bit(i, toclear); - if (severity == MCE_NO_SEVERITY) { - /* - * Machine check event was not enabled. Clear, but - * ignore. - */ - continue; - } - - mce_read_aux(&m, i); - - /* assuming valid severity level != 0 */ - m.severity = severity; - - mce_log(&m); - - if (severity > worst) { - *final = m; - worst = severity; - } - } - - /* mce_clear_state will clear *final, save locally for use later */ - m = *final; + __mc_scan_banks(&m, final, toclear, valid_banks, no_way_out, &worst); if (!no_way_out) mce_clear_state(toclear); @@ -1319,7 +1331,7 @@ void do_machine_check(struct pt_regs *regs, long error_code) if (worst > 0) mce_report_event(regs); mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); -out: + sync_core(); if (worst != MCE_AR_SEVERITY && !kill_it) @@ -2165,9 +2177,6 @@ static ssize_t store_int_with_restart(struct device *s, if (check_interval == old_check_interval) return ret; - if (check_interval < 1) - check_interval = 1; - mutex_lock(&mce_sysfs_mutex); mce_restart(); mutex_unlock(&mce_sysfs_mutex); diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 08286269fd24..b9bc8a1a584e 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -509,12 +509,20 @@ static struct platform_device *microcode_pdev; static int check_online_cpus(void) { - if (num_online_cpus() == num_present_cpus()) - return 0; + unsigned int cpu; - pr_err("Not all CPUs online, aborting microcode update.\n"); + /* + * Make sure all CPUs are online. It's fine for SMT to be disabled if + * all the primary threads are still online. + */ + for_each_present_cpu(cpu) { + if (topology_is_primary_thread(cpu) && !cpu_online(cpu)) { + pr_err("Not all CPUs online, aborting microcode update.\n"); + return -EINVAL; + } + } - return -EINVAL; + return 0; } static atomic_t late_cpus_in; diff --git a/arch/x86/kernel/cpu/mtrr/if.c b/arch/x86/kernel/cpu/mtrr/if.c index 4021d3859499..40eee6cc4124 100644 --- a/arch/x86/kernel/cpu/mtrr/if.c +++ b/arch/x86/kernel/cpu/mtrr/if.c @@ -106,7 +106,8 @@ mtrr_write(struct file *file, const char __user *buf, size_t len, loff_t * ppos) memset(line, 0, LINE_SIZE); - length = strncpy_from_user(line, buf, LINE_SIZE - 1); + len = min_t(size_t, len, LINE_SIZE - 1); + length = strncpy_from_user(line, buf, len); if (length < 0) return length; diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 81c0afb39d0a..71ca064e3794 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -22,18 +22,10 @@ #define BITS_SHIFT_NEXT_LEVEL(eax) ((eax) & 0x1f) #define LEVEL_MAX_SIBLINGS(ebx) ((ebx) & 0xffff) -/* - * Check for extended topology enumeration cpuid leaf 0xb and if it - * exists, use it for populating initial_apicid and cpu topology - * detection. - */ -int detect_extended_topology(struct cpuinfo_x86 *c) +int detect_extended_topology_early(struct cpuinfo_x86 *c) { #ifdef CONFIG_SMP - unsigned int eax, ebx, ecx, edx, sub_index; - unsigned int ht_mask_width, core_plus_mask_width; - unsigned int core_select_mask, core_level_siblings; - static bool printed; + unsigned int eax, ebx, ecx, edx; if (c->cpuid_level < 0xb) return -1; @@ -52,10 +44,30 @@ int detect_extended_topology(struct cpuinfo_x86 *c) * initial apic id, which also represents 32-bit extended x2apic id. */ c->initial_apicid = edx; + smp_num_siblings = LEVEL_MAX_SIBLINGS(ebx); +#endif + return 0; +} + +/* + * Check for extended topology enumeration cpuid leaf 0xb and if it + * exists, use it for populating initial_apicid and cpu topology + * detection. + */ +int detect_extended_topology(struct cpuinfo_x86 *c) +{ +#ifdef CONFIG_SMP + unsigned int eax, ebx, ecx, edx, sub_index; + unsigned int ht_mask_width, core_plus_mask_width; + unsigned int core_select_mask, core_level_siblings; + + if (detect_extended_topology_early(c) < 0) + return -1; /* * Populate HT related information from sub-leaf level 0. */ + cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx); core_level_siblings = smp_num_siblings = LEVEL_MAX_SIBLINGS(ebx); core_plus_mask_width = ht_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); @@ -86,15 +98,6 @@ int detect_extended_topology(struct cpuinfo_x86 *c) c->apicid = apic->phys_pkg_id(c->initial_apicid, 0); c->x86_max_cores = (core_level_siblings / smp_num_siblings); - - if (!printed) { - pr_info("CPU: Physical Processor ID: %d\n", - c->phys_proc_id); - if (c->x86_max_cores > 1) - pr_info("CPU: Processor Core ID: %d\n", - c->cpu_core_id); - printed = 1; - } #endif return 0; } diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 666a284116ac..9c8652974f8e 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -22,8 +22,6 @@ #include <asm/stacktrace.h> #include <asm/unwind.h> -#define OPCODE_BUFSIZE 64 - int panic_on_unrecovered_nmi; int panic_on_io_nmi; static int die_counter; @@ -93,26 +91,18 @@ static void printk_stack_address(unsigned long address, int reliable, */ void show_opcodes(u8 *rip, const char *loglvl) { - unsigned int code_prologue = OPCODE_BUFSIZE * 2 / 3; +#define PROLOGUE_SIZE 42 +#define EPILOGUE_SIZE 21 +#define OPCODE_BUFSIZE (PROLOGUE_SIZE + 1 + EPILOGUE_SIZE) u8 opcodes[OPCODE_BUFSIZE]; - u8 *ip; - int i; - - printk("%sCode: ", loglvl); - - ip = (u8 *)rip - code_prologue; - if (probe_kernel_read(opcodes, ip, OPCODE_BUFSIZE)) { - pr_cont("Bad RIP value.\n"); - return; - } - for (i = 0; i < OPCODE_BUFSIZE; i++, ip++) { - if (ip == rip) - pr_cont("<%02x> ", opcodes[i]); - else - pr_cont("%02x ", opcodes[i]); + if (probe_kernel_read(opcodes, rip - PROLOGUE_SIZE, OPCODE_BUFSIZE)) { + printk("%sCode: Bad RIP value.\n", loglvl); + } else { + printk("%sCode: %" __stringify(PROLOGUE_SIZE) "ph <%02x> %" + __stringify(EPILOGUE_SIZE) "ph\n", loglvl, opcodes, + opcodes[PROLOGUE_SIZE], opcodes + PROLOGUE_SIZE + 1); } - pr_cont("\n"); } void show_ip(struct pt_regs *regs, const char *loglvl) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index f92a6593de1e..2ea85b32421a 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -10,6 +10,7 @@ #include <asm/fpu/signal.h> #include <asm/fpu/types.h> #include <asm/traps.h> +#include <asm/irq_regs.h> #include <linux/hardirq.h> #include <linux/pkeys.h> diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S index abe6df15a8fb..30f9cb2c0b55 100644 --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -512,11 +512,18 @@ ENTRY(initial_code) ENTRY(setup_once_ref) .long setup_once +#ifdef CONFIG_PAGE_TABLE_ISOLATION +#define PGD_ALIGN (2 * PAGE_SIZE) +#define PTI_USER_PGD_FILL 1024 +#else +#define PGD_ALIGN (PAGE_SIZE) +#define PTI_USER_PGD_FILL 0 +#endif /* * BSS section */ __PAGE_ALIGNED_BSS - .align PAGE_SIZE + .align PGD_ALIGN #ifdef CONFIG_X86_PAE .globl initial_pg_pmd initial_pg_pmd: @@ -526,14 +533,17 @@ initial_pg_pmd: initial_page_table: .fill 1024,4,0 #endif + .align PGD_ALIGN initial_pg_fixmap: .fill 1024,4,0 -.globl empty_zero_page -empty_zero_page: - .fill 4096,1,0 .globl swapper_pg_dir + .align PGD_ALIGN swapper_pg_dir: .fill 1024,4,0 + .fill PTI_USER_PGD_FILL,4,0 +.globl empty_zero_page +empty_zero_page: + .fill 4096,1,0 EXPORT_SYMBOL(empty_zero_page) /* @@ -542,7 +552,7 @@ EXPORT_SYMBOL(empty_zero_page) #ifdef CONFIG_X86_PAE __PAGE_ALIGNED_DATA /* Page-aligned for the benefit of paravirt? */ - .align PAGE_SIZE + .align PGD_ALIGN ENTRY(initial_page_table) .long pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */ # if KPMDS == 3 diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 8344dd2f310a..15ebc2fc166e 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -235,7 +235,7 @@ ENTRY(secondary_startup_64) * address given in m16:64. */ pushq $.Lafter_lret # put return address on stack for unwinder - xorq %rbp, %rbp # clear frame pointer + xorl %ebp, %ebp # clear frame pointer movq initial_code(%rip), %rax pushq $__KERNEL_CS # set correct cs pushq %rax # target address in negative space diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 346b24883911..b0acb22e5a46 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -1,6 +1,7 @@ #include <linux/clocksource.h> #include <linux/clockchips.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/export.h> #include <linux/delay.h> #include <linux/errno.h> diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoint.c index 8771766d46b6..34a5c1715148 100644 --- a/arch/x86/kernel/hw_breakpoint.c +++ b/arch/x86/kernel/hw_breakpoint.c @@ -169,28 +169,29 @@ void arch_uninstall_hw_breakpoint(struct perf_event *bp) set_dr_addr_mask(0, i); } -/* - * Check for virtual address in kernel space. - */ -int arch_check_bp_in_kernelspace(struct perf_event *bp) +static int arch_bp_generic_len(int x86_len) { - unsigned int len; - unsigned long va; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - - va = info->address; - len = bp->attr.bp_len; - - /* - * We don't need to worry about va + len - 1 overflowing: - * we already require that va is aligned to a multiple of len. - */ - return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX); + switch (x86_len) { + case X86_BREAKPOINT_LEN_1: + return HW_BREAKPOINT_LEN_1; + case X86_BREAKPOINT_LEN_2: + return HW_BREAKPOINT_LEN_2; + case X86_BREAKPOINT_LEN_4: + return HW_BREAKPOINT_LEN_4; +#ifdef CONFIG_X86_64 + case X86_BREAKPOINT_LEN_8: + return HW_BREAKPOINT_LEN_8; +#endif + default: + return -EINVAL; + } } int arch_bp_generic_fields(int x86_len, int x86_type, int *gen_len, int *gen_type) { + int len; + /* Type */ switch (x86_type) { case X86_BREAKPOINT_EXECUTE: @@ -211,42 +212,47 @@ int arch_bp_generic_fields(int x86_len, int x86_type, } /* Len */ - switch (x86_len) { - case X86_BREAKPOINT_LEN_1: - *gen_len = HW_BREAKPOINT_LEN_1; - break; - case X86_BREAKPOINT_LEN_2: - *gen_len = HW_BREAKPOINT_LEN_2; - break; - case X86_BREAKPOINT_LEN_4: - *gen_len = HW_BREAKPOINT_LEN_4; - break; -#ifdef CONFIG_X86_64 - case X86_BREAKPOINT_LEN_8: - *gen_len = HW_BREAKPOINT_LEN_8; - break; -#endif - default: + len = arch_bp_generic_len(x86_len); + if (len < 0) return -EINVAL; - } + *gen_len = len; return 0; } - -static int arch_build_bp_info(struct perf_event *bp) +/* + * Check for virtual address in kernel space. + */ +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); + unsigned long va; + int len; - info->address = bp->attr.bp_addr; + va = hw->address; + len = arch_bp_generic_len(hw->len); + WARN_ON_ONCE(len < 0); + + /* + * We don't need to worry about va + len - 1 overflowing: + * we already require that va is aligned to a multiple of len. + */ + return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX); +} + +static int arch_build_bp_info(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) +{ + hw->address = attr->bp_addr; + hw->mask = 0; /* Type */ - switch (bp->attr.bp_type) { + switch (attr->bp_type) { case HW_BREAKPOINT_W: - info->type = X86_BREAKPOINT_WRITE; + hw->type = X86_BREAKPOINT_WRITE; break; case HW_BREAKPOINT_W | HW_BREAKPOINT_R: - info->type = X86_BREAKPOINT_RW; + hw->type = X86_BREAKPOINT_RW; break; case HW_BREAKPOINT_X: /* @@ -254,23 +260,23 @@ static int arch_build_bp_info(struct perf_event *bp) * acceptable for kprobes. On non-kprobes kernels, we don't * allow kernel breakpoints at all. */ - if (bp->attr.bp_addr >= TASK_SIZE_MAX) { + if (attr->bp_addr >= TASK_SIZE_MAX) { #ifdef CONFIG_KPROBES - if (within_kprobe_blacklist(bp->attr.bp_addr)) + if (within_kprobe_blacklist(attr->bp_addr)) return -EINVAL; #else return -EINVAL; #endif } - info->type = X86_BREAKPOINT_EXECUTE; + hw->type = X86_BREAKPOINT_EXECUTE; /* * x86 inst breakpoints need to have a specific undefined len. * But we still need to check userspace is not trying to setup * an unsupported length, to get a range breakpoint for example. */ - if (bp->attr.bp_len == sizeof(long)) { - info->len = X86_BREAKPOINT_LEN_X; + if (attr->bp_len == sizeof(long)) { + hw->len = X86_BREAKPOINT_LEN_X; return 0; } default: @@ -278,28 +284,26 @@ static int arch_build_bp_info(struct perf_event *bp) } /* Len */ - info->mask = 0; - - switch (bp->attr.bp_len) { + switch (attr->bp_len) { case HW_BREAKPOINT_LEN_1: - info->len = X86_BREAKPOINT_LEN_1; + hw->len = X86_BREAKPOINT_LEN_1; break; case HW_BREAKPOINT_LEN_2: - info->len = X86_BREAKPOINT_LEN_2; + hw->len = X86_BREAKPOINT_LEN_2; break; case HW_BREAKPOINT_LEN_4: - info->len = X86_BREAKPOINT_LEN_4; + hw->len = X86_BREAKPOINT_LEN_4; break; #ifdef CONFIG_X86_64 case HW_BREAKPOINT_LEN_8: - info->len = X86_BREAKPOINT_LEN_8; + hw->len = X86_BREAKPOINT_LEN_8; break; #endif default: /* AMD range breakpoint */ - if (!is_power_of_2(bp->attr.bp_len)) + if (!is_power_of_2(attr->bp_len)) return -EINVAL; - if (bp->attr.bp_addr & (bp->attr.bp_len - 1)) + if (attr->bp_addr & (attr->bp_len - 1)) return -EINVAL; if (!boot_cpu_has(X86_FEATURE_BPEXT)) @@ -312,8 +316,8 @@ static int arch_build_bp_info(struct perf_event *bp) * breakpoints, then we'll have to check for kprobe-blacklisted * addresses anywhere in the range. */ - info->mask = bp->attr.bp_len - 1; - info->len = X86_BREAKPOINT_LEN_1; + hw->mask = attr->bp_len - 1; + hw->len = X86_BREAKPOINT_LEN_1; } return 0; @@ -322,22 +326,23 @@ static int arch_build_bp_info(struct perf_event *bp) /* * Validate the arch-specific HW Breakpoint register settings */ -int arch_validate_hwbkpt_settings(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); unsigned int align; int ret; - ret = arch_build_bp_info(bp); + ret = arch_build_bp_info(bp, attr, hw); if (ret) return ret; - switch (info->len) { + switch (hw->len) { case X86_BREAKPOINT_LEN_1: align = 0; - if (info->mask) - align = info->mask; + if (hw->mask) + align = hw->mask; break; case X86_BREAKPOINT_LEN_2: align = 1; @@ -358,7 +363,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) * Check that the low-order bits of the address are appropriate * for the alignment implied by len. */ - if (info->address & align) + if (hw->address & align) return -EINVAL; return 0; diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c index 86c4439f9d74..519649ddf100 100644 --- a/arch/x86/kernel/i8259.c +++ b/arch/x86/kernel/i8259.c @@ -5,6 +5,7 @@ #include <linux/sched.h> #include <linux/ioport.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/timex.h> #include <linux/random.h> #include <linux/init.h> diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 74383a3780dc..01adea278a71 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -8,6 +8,7 @@ #include <asm/traps.h> #include <asm/proto.h> #include <asm/desc.h> +#include <asm/hw_irq.h> struct idt_data { unsigned int vector; diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 328d027d829d..59b5f2ea7c2f 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -10,6 +10,7 @@ #include <linux/ftrace.h> #include <linux/delay.h> #include <linux/export.h> +#include <linux/irq.h> #include <asm/apic.h> #include <asm/io_apic.h> diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index c1bdbd3d3232..95600a99ae93 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -11,6 +11,7 @@ #include <linux/seq_file.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/kernel_stat.h> #include <linux/notifier.h> #include <linux/cpu.h> diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index d86e344f5b3d..0469cd078db1 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -11,6 +11,7 @@ #include <linux/kernel_stat.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/seq_file.h> #include <linux/delay.h> #include <linux/ftrace.h> diff --git a/arch/x86/kernel/irqflags.S b/arch/x86/kernel/irqflags.S new file mode 100644 index 000000000000..ddeeaac8adda --- /dev/null +++ b/arch/x86/kernel/irqflags.S @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <asm/asm.h> +#include <asm/export.h> +#include <linux/linkage.h> + +/* + * unsigned long native_save_fl(void) + */ +ENTRY(native_save_fl) + pushf + pop %_ASM_AX + ret +ENDPROC(native_save_fl) +EXPORT_SYMBOL(native_save_fl) + +/* + * void native_restore_fl(unsigned long flags) + * %eax/%rdi: flags + */ +ENTRY(native_restore_fl) + push %_ASM_ARG1 + popf + ret +ENDPROC(native_restore_fl) +EXPORT_SYMBOL(native_restore_fl) diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index 772196c1b8c4..a0693b71cfc1 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -5,6 +5,7 @@ #include <linux/sched.h> #include <linux/ioport.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/timex.h> #include <linux/random.h> #include <linux/kprobes.h> diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index e56c95be2808..eeea935e9bb5 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -37,15 +37,18 @@ static void bug_at(unsigned char *ip, int line) BUG(); } -static void __jump_label_transform(struct jump_entry *entry, - enum jump_label_type type, - void *(*poker)(void *, const void *, size_t), - int init) +static void __ref __jump_label_transform(struct jump_entry *entry, + enum jump_label_type type, + void *(*poker)(void *, const void *, size_t), + int init) { union jump_code_union code; const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; + if (early_boot_irqs_disabled) + poker = text_poke_early; + if (type == JUMP_LABEL_JMP) { if (init) { /* diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h index ae38dccf0c8f..2b949f4fd4d8 100644 --- a/arch/x86/kernel/kprobes/common.h +++ b/arch/x86/kernel/kprobes/common.h @@ -105,14 +105,4 @@ static inline unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsig } #endif -#ifdef CONFIG_KPROBES_ON_FTRACE -extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb); -#else -static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb) -{ - return 0; -} -#endif #endif diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 6f4d42377fe5..b0d1e81c96bb 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -66,8 +66,6 @@ #include "common.h" -void jprobe_return_end(void); - DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); @@ -395,8 +393,6 @@ int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn) - (u8 *) real; if ((s64) (s32) newdisp != newdisp) { pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); - pr_err("\tSrc: %p, Dest: %p, old disp: %x\n", - src, real, insn->displacement.value); return 0; } disp = (u8 *) dest + insn_offset_displacement(insn); @@ -596,7 +592,6 @@ static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, * stepping. */ regs->ip = (unsigned long)p->ainsn.insn; - preempt_enable_no_resched(); return; } #endif @@ -640,8 +635,7 @@ static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, * Raise a BUG or we'll continue in an endless reentering loop * and eventually a stack overflow. */ - printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n", - p->addr); + pr_err("Unrecoverable kprobe detected.\n"); dump_kprobe(p); BUG(); default: @@ -669,12 +663,10 @@ int kprobe_int3_handler(struct pt_regs *regs) addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); /* - * We don't want to be preempted for the entire - * duration of kprobe processing. We conditionally - * re-enable preemption at the end of this function, - * and also in reenter_kprobe() and setup_singlestep(). + * We don't want to be preempted for the entire duration of kprobe + * processing. Since int3 and debug trap disables irqs and we clear + * IF while singlestepping, it must be no preemptible. */ - preempt_disable(); kcb = get_kprobe_ctlblk(); p = get_kprobe(addr); @@ -690,13 +682,14 @@ int kprobe_int3_handler(struct pt_regs *regs) /* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a - * pre-handler and it returned non-zero, it prepped - * for calling the break_handler below on re-entry - * for jprobe processing, so get out doing nothing - * more here. + * pre-handler and it returned non-zero, that means + * user handler setup registers to exit to another + * instruction, we must skip the single stepping. */ if (!p->pre_handler || !p->pre_handler(p, regs)) setup_singlestep(p, regs, kcb, 0); + else + reset_current_kprobe(); return 1; } } else if (*addr != BREAKPOINT_INSTRUCTION) { @@ -710,18 +703,9 @@ int kprobe_int3_handler(struct pt_regs *regs) * the original instruction. */ regs->ip = (unsigned long)addr; - preempt_enable_no_resched(); return 1; - } else if (kprobe_running()) { - p = __this_cpu_read(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) { - if (!skip_singlestep(p, regs, kcb)) - setup_singlestep(p, regs, kcb, 0); - return 1; - } } /* else: not a kprobe fault; let the kernel handle it */ - preempt_enable_no_resched(); return 0; } NOKPROBE_SYMBOL(kprobe_int3_handler); @@ -972,8 +956,6 @@ int kprobe_debug_handler(struct pt_regs *regs) } reset_current_kprobe(); out: - preempt_enable_no_resched(); - /* * if somebody else is singlestepping across a probe point, flags * will have TF set, in which case, continue the remaining processing @@ -1020,7 +1002,6 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr) restore_previous_kprobe(kcb); else reset_current_kprobe(); - preempt_enable_no_resched(); } else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE || kcb->kprobe_status == KPROBE_HIT_SSDONE) { /* @@ -1083,93 +1064,6 @@ int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, } NOKPROBE_SYMBOL(kprobe_exceptions_notify); -int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - unsigned long addr; - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - kcb->jprobe_saved_regs = *regs; - kcb->jprobe_saved_sp = stack_addr(regs); - addr = (unsigned long)(kcb->jprobe_saved_sp); - - /* - * As Linus pointed out, gcc assumes that the callee - * owns the argument space and could overwrite it, e.g. - * tailcall optimization. So, to be absolutely safe - * we also save and restore enough stack bytes to cover - * the argument area. - * Use __memcpy() to avoid KASAN stack out-of-bounds reports as we copy - * raw stack chunk with redzones: - */ - __memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr, MIN_STACK_SIZE(addr)); - regs->ip = (unsigned long)(jp->entry); - - /* - * jprobes use jprobe_return() which skips the normal return - * path of the function, and this messes up the accounting of the - * function graph tracer to get messed up. - * - * Pause function graph tracing while performing the jprobe function. - */ - pause_graph_tracing(); - return 1; -} -NOKPROBE_SYMBOL(setjmp_pre_handler); - -void jprobe_return(void) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - /* Unpoison stack redzones in the frames we are going to jump over. */ - kasan_unpoison_stack_above_sp_to(kcb->jprobe_saved_sp); - - asm volatile ( -#ifdef CONFIG_X86_64 - " xchg %%rbx,%%rsp \n" -#else - " xchgl %%ebx,%%esp \n" -#endif - " int3 \n" - " .globl jprobe_return_end\n" - " jprobe_return_end: \n" - " nop \n"::"b" - (kcb->jprobe_saved_sp):"memory"); -} -NOKPROBE_SYMBOL(jprobe_return); -NOKPROBE_SYMBOL(jprobe_return_end); - -int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - u8 *addr = (u8 *) (regs->ip - 1); - struct jprobe *jp = container_of(p, struct jprobe, kp); - void *saved_sp = kcb->jprobe_saved_sp; - - if ((addr > (u8 *) jprobe_return) && - (addr < (u8 *) jprobe_return_end)) { - if (stack_addr(regs) != saved_sp) { - struct pt_regs *saved_regs = &kcb->jprobe_saved_regs; - printk(KERN_ERR - "current sp %p does not match saved sp %p\n", - stack_addr(regs), saved_sp); - printk(KERN_ERR "Saved registers for jprobe %p\n", jp); - show_regs(saved_regs); - printk(KERN_ERR "Current registers\n"); - show_regs(regs); - BUG(); - } - /* It's OK to start function graph tracing again */ - unpause_graph_tracing(); - *regs = kcb->jprobe_saved_regs; - __memcpy(saved_sp, kcb->jprobes_stack, MIN_STACK_SIZE(saved_sp)); - preempt_enable_no_resched(); - return 1; - } - return 0; -} -NOKPROBE_SYMBOL(longjmp_break_handler); - bool arch_within_kprobe_blacklist(unsigned long addr) { bool is_in_entry_trampoline_section = false; diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c index 8dc0161cec8f..ef819e19650b 100644 --- a/arch/x86/kernel/kprobes/ftrace.c +++ b/arch/x86/kernel/kprobes/ftrace.c @@ -25,36 +25,6 @@ #include "common.h" -static nokprobe_inline -void __skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb, unsigned long orig_ip) -{ - /* - * Emulate singlestep (and also recover regs->ip) - * as if there is a 5byte nop - */ - regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE; - if (unlikely(p->post_handler)) { - kcb->kprobe_status = KPROBE_HIT_SSDONE; - p->post_handler(p, regs, 0); - } - __this_cpu_write(current_kprobe, NULL); - if (orig_ip) - regs->ip = orig_ip; -} - -int skip_singlestep(struct kprobe *p, struct pt_regs *regs, - struct kprobe_ctlblk *kcb) -{ - if (kprobe_ftrace(p)) { - __skip_singlestep(p, regs, kcb, 0); - preempt_enable_no_resched(); - return 1; - } - return 0; -} -NOKPROBE_SYMBOL(skip_singlestep); - /* Ftrace callback handler for kprobes -- called under preepmt disabed */ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct pt_regs *regs) @@ -75,18 +45,25 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */ regs->ip = ip + sizeof(kprobe_opcode_t); - /* To emulate trap based kprobes, preempt_disable here */ - preempt_disable(); __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (!p->pre_handler || !p->pre_handler(p, regs)) { - __skip_singlestep(p, regs, kcb, orig_ip); - preempt_enable_no_resched(); + /* + * Emulate singlestep (and also recover regs->ip) + * as if there is a 5byte nop + */ + regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE; + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + regs->ip = orig_ip; } /* - * If pre_handler returns !0, it sets regs->ip and - * resets current kprobe, and keep preempt count +1. + * If pre_handler returns !0, it changes regs->ip. We have to + * skip emulating post_handler. */ + __this_cpu_write(current_kprobe, NULL); } } NOKPROBE_SYMBOL(kprobe_ftrace_handler); diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 203d398802a3..eaf02f2e7300 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -491,7 +491,6 @@ int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter) regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX; if (!reenter) reset_current_kprobe(); - preempt_enable_no_resched(); return 1; } return 0; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 5b2300b818af..09aaabb2bbf1 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -45,7 +45,6 @@ #include <asm/apic.h> #include <asm/apicdef.h> #include <asm/hypervisor.h> -#include <asm/kvm_guest.h> static int kvmapf = 1; @@ -66,15 +65,6 @@ static int __init parse_no_stealacc(char *arg) early_param("no-steal-acc", parse_no_stealacc); -static int kvmclock_vsyscall = 1; -static int __init parse_no_kvmclock_vsyscall(char *arg) -{ - kvmclock_vsyscall = 0; - return 0; -} - -early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall); - static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); static DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64); static int has_steal_clock = 0; @@ -154,7 +144,7 @@ void kvm_async_pf_task_wait(u32 token, int interrupt_kernel) for (;;) { if (!n.halted) - prepare_to_swait(&n.wq, &wait, TASK_UNINTERRUPTIBLE); + prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE); if (hlist_unhashed(&n.link)) break; @@ -188,7 +178,7 @@ static void apf_task_wake_one(struct kvm_task_sleep_node *n) if (n->halted) smp_send_reschedule(n->cpu); else if (swq_has_sleeper(&n->wq)) - swake_up(&n->wq); + swake_up_one(&n->wq); } static void apf_task_wake_all(void) @@ -560,9 +550,6 @@ static void __init kvm_guest_init(void) if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) apic_set_eoi_write(kvm_guest_apic_eoi_write); - if (kvmclock_vsyscall) - kvm_setup_vsyscall_timeinfo(); - #ifdef CONFIG_SMP smp_ops.smp_prepare_cpus = kvm_smp_prepare_cpus; smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; @@ -628,6 +615,7 @@ const __initconst struct hypervisor_x86 x86_hyper_kvm = { .name = "KVM", .detect = kvm_detect, .type = X86_HYPER_KVM, + .init.init_platform = kvmclock_init, .init.guest_late_init = kvm_guest_init, .init.x2apic_available = kvm_para_available, }; diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index bf8d1eb7fca3..1e6764648af3 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -23,30 +23,57 @@ #include <asm/apic.h> #include <linux/percpu.h> #include <linux/hardirq.h> -#include <linux/memblock.h> +#include <linux/cpuhotplug.h> #include <linux/sched.h> #include <linux/sched/clock.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <asm/hypervisor.h> #include <asm/mem_encrypt.h> #include <asm/x86_init.h> #include <asm/reboot.h> #include <asm/kvmclock.h> -static int kvmclock __ro_after_init = 1; -static int msr_kvm_system_time = MSR_KVM_SYSTEM_TIME; -static int msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK; -static u64 kvm_sched_clock_offset; +static int kvmclock __initdata = 1; +static int kvmclock_vsyscall __initdata = 1; +static int msr_kvm_system_time __ro_after_init = MSR_KVM_SYSTEM_TIME; +static int msr_kvm_wall_clock __ro_after_init = MSR_KVM_WALL_CLOCK; +static u64 kvm_sched_clock_offset __ro_after_init; -static int parse_no_kvmclock(char *arg) +static int __init parse_no_kvmclock(char *arg) { kvmclock = 0; return 0; } early_param("no-kvmclock", parse_no_kvmclock); -/* The hypervisor will put information about time periodically here */ -static struct pvclock_vsyscall_time_info *hv_clock; -static struct pvclock_wall_clock *wall_clock; +static int __init parse_no_kvmclock_vsyscall(char *arg) +{ + kvmclock_vsyscall = 0; + return 0; +} +early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall); + +/* Aligned to page sizes to match whats mapped via vsyscalls to userspace */ +#define HV_CLOCK_SIZE (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS) +#define HVC_BOOT_ARRAY_SIZE \ + (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info)) + +static struct pvclock_vsyscall_time_info + hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE); +static struct pvclock_wall_clock wall_clock; +static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu); + +static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void) +{ + return &this_cpu_read(hv_clock_per_cpu)->pvti; +} + +static inline struct pvclock_vsyscall_time_info *this_cpu_hvclock(void) +{ + return this_cpu_read(hv_clock_per_cpu); +} /* * The wallclock is the time of day when we booted. Since then, some time may @@ -55,21 +82,10 @@ static struct pvclock_wall_clock *wall_clock; */ static void kvm_get_wallclock(struct timespec64 *now) { - struct pvclock_vcpu_time_info *vcpu_time; - int low, high; - int cpu; - - low = (int)slow_virt_to_phys(wall_clock); - high = ((u64)slow_virt_to_phys(wall_clock) >> 32); - - native_write_msr(msr_kvm_wall_clock, low, high); - - cpu = get_cpu(); - - vcpu_time = &hv_clock[cpu].pvti; - pvclock_read_wallclock(wall_clock, vcpu_time, now); - - put_cpu(); + wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(&wall_clock)); + preempt_disable(); + pvclock_read_wallclock(&wall_clock, this_cpu_pvti(), now); + preempt_enable(); } static int kvm_set_wallclock(const struct timespec64 *now) @@ -79,14 +95,10 @@ static int kvm_set_wallclock(const struct timespec64 *now) static u64 kvm_clock_read(void) { - struct pvclock_vcpu_time_info *src; u64 ret; - int cpu; preempt_disable_notrace(); - cpu = smp_processor_id(); - src = &hv_clock[cpu].pvti; - ret = pvclock_clocksource_read(src); + ret = pvclock_clocksource_read(this_cpu_pvti()); preempt_enable_notrace(); return ret; } @@ -112,11 +124,11 @@ static inline void kvm_sched_clock_init(bool stable) kvm_sched_clock_offset = kvm_clock_read(); pv_time_ops.sched_clock = kvm_sched_clock_read; - printk(KERN_INFO "kvm-clock: using sched offset of %llu cycles\n", - kvm_sched_clock_offset); + pr_info("kvm-clock: using sched offset of %llu cycles", + kvm_sched_clock_offset); BUILD_BUG_ON(sizeof(kvm_sched_clock_offset) > - sizeof(((struct pvclock_vcpu_time_info *)NULL)->system_time)); + sizeof(((struct pvclock_vcpu_time_info *)NULL)->system_time)); } /* @@ -130,18 +142,11 @@ static inline void kvm_sched_clock_init(bool stable) */ static unsigned long kvm_get_tsc_khz(void) { - struct pvclock_vcpu_time_info *src; - int cpu; - unsigned long tsc_khz; - - cpu = get_cpu(); - src = &hv_clock[cpu].pvti; - tsc_khz = pvclock_tsc_khz(src); - put_cpu(); - return tsc_khz; + setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); + return pvclock_tsc_khz(this_cpu_pvti()); } -static void kvm_get_preset_lpj(void) +static void __init kvm_get_preset_lpj(void) { unsigned long khz; u64 lpj; @@ -155,49 +160,40 @@ static void kvm_get_preset_lpj(void) bool kvm_check_and_clear_guest_paused(void) { + struct pvclock_vsyscall_time_info *src = this_cpu_hvclock(); bool ret = false; - struct pvclock_vcpu_time_info *src; - int cpu = smp_processor_id(); - if (!hv_clock) + if (!src) return ret; - src = &hv_clock[cpu].pvti; - if ((src->flags & PVCLOCK_GUEST_STOPPED) != 0) { - src->flags &= ~PVCLOCK_GUEST_STOPPED; + if ((src->pvti.flags & PVCLOCK_GUEST_STOPPED) != 0) { + src->pvti.flags &= ~PVCLOCK_GUEST_STOPPED; pvclock_touch_watchdogs(); ret = true; } - return ret; } struct clocksource kvm_clock = { - .name = "kvm-clock", - .read = kvm_clock_get_cycles, - .rating = 400, - .mask = CLOCKSOURCE_MASK(64), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .name = "kvm-clock", + .read = kvm_clock_get_cycles, + .rating = 400, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; EXPORT_SYMBOL_GPL(kvm_clock); -int kvm_register_clock(char *txt) +static void kvm_register_clock(char *txt) { - int cpu = smp_processor_id(); - int low, high, ret; - struct pvclock_vcpu_time_info *src; - - if (!hv_clock) - return 0; + struct pvclock_vsyscall_time_info *src = this_cpu_hvclock(); + u64 pa; - src = &hv_clock[cpu].pvti; - low = (int)slow_virt_to_phys(src) | 1; - high = ((u64)slow_virt_to_phys(src) >> 32); - ret = native_write_msr_safe(msr_kvm_system_time, low, high); - printk(KERN_INFO "kvm-clock: cpu %d, msr %x:%x, %s\n", - cpu, high, low, txt); + if (!src) + return; - return ret; + pa = slow_virt_to_phys(&src->pvti) | 0x01ULL; + wrmsrl(msr_kvm_system_time, pa); + pr_info("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt); } static void kvm_save_sched_clock_state(void) @@ -212,11 +208,7 @@ static void kvm_restore_sched_clock_state(void) #ifdef CONFIG_X86_LOCAL_APIC static void kvm_setup_secondary_clock(void) { - /* - * Now that the first cpu already had this clocksource initialized, - * we shouldn't fail. - */ - WARN_ON(kvm_register_clock("secondary cpu clock")); + kvm_register_clock("secondary cpu clock"); } #endif @@ -244,98 +236,84 @@ static void kvm_shutdown(void) native_machine_shutdown(); } -static phys_addr_t __init kvm_memblock_alloc(phys_addr_t size, - phys_addr_t align) +static int __init kvm_setup_vsyscall_timeinfo(void) { - phys_addr_t mem; +#ifdef CONFIG_X86_64 + u8 flags; - mem = memblock_alloc(size, align); - if (!mem) + if (!per_cpu(hv_clock_per_cpu, 0) || !kvmclock_vsyscall) return 0; - if (sev_active()) { - if (early_set_memory_decrypted((unsigned long)__va(mem), size)) - goto e_free; - } + flags = pvclock_read_flags(&hv_clock_boot[0].pvti); + if (!(flags & PVCLOCK_TSC_STABLE_BIT)) + return 0; - return mem; -e_free: - memblock_free(mem, size); + kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; +#endif return 0; } +early_initcall(kvm_setup_vsyscall_timeinfo); -static void __init kvm_memblock_free(phys_addr_t addr, phys_addr_t size) +static int kvmclock_setup_percpu(unsigned int cpu) { - if (sev_active()) - early_set_memory_encrypted((unsigned long)__va(addr), size); + struct pvclock_vsyscall_time_info *p = per_cpu(hv_clock_per_cpu, cpu); + + /* + * The per cpu area setup replicates CPU0 data to all cpu + * pointers. So carefully check. CPU0 has been set up in init + * already. + */ + if (!cpu || (p && p != per_cpu(hv_clock_per_cpu, 0))) + return 0; + + /* Use the static page for the first CPUs, allocate otherwise */ + if (cpu < HVC_BOOT_ARRAY_SIZE) + p = &hv_clock_boot[cpu]; + else + p = kzalloc(sizeof(*p), GFP_KERNEL); - memblock_free(addr, size); + per_cpu(hv_clock_per_cpu, cpu) = p; + return p ? 0 : -ENOMEM; } void __init kvmclock_init(void) { - struct pvclock_vcpu_time_info *vcpu_time; - unsigned long mem, mem_wall_clock; - int size, cpu, wall_clock_size; u8 flags; - size = PAGE_ALIGN(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS); - - if (!kvm_para_available()) + if (!kvm_para_available() || !kvmclock) return; - if (kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) { + if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) { msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW; msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW; - } else if (!(kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE))) - return; - - wall_clock_size = PAGE_ALIGN(sizeof(struct pvclock_wall_clock)); - mem_wall_clock = kvm_memblock_alloc(wall_clock_size, PAGE_SIZE); - if (!mem_wall_clock) - return; - - wall_clock = __va(mem_wall_clock); - memset(wall_clock, 0, wall_clock_size); - - mem = kvm_memblock_alloc(size, PAGE_SIZE); - if (!mem) { - kvm_memblock_free(mem_wall_clock, wall_clock_size); - wall_clock = NULL; + } else if (!kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)) { return; } - hv_clock = __va(mem); - memset(hv_clock, 0, size); - - if (kvm_register_clock("primary cpu clock")) { - hv_clock = NULL; - kvm_memblock_free(mem, size); - kvm_memblock_free(mem_wall_clock, wall_clock_size); - wall_clock = NULL; + if (cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "kvmclock:setup_percpu", + kvmclock_setup_percpu, NULL) < 0) { return; } - printk(KERN_INFO "kvm-clock: Using msrs %x and %x", + pr_info("kvm-clock: Using msrs %x and %x", msr_kvm_system_time, msr_kvm_wall_clock); + this_cpu_write(hv_clock_per_cpu, &hv_clock_boot[0]); + kvm_register_clock("primary cpu clock"); + pvclock_set_pvti_cpu0_va(hv_clock_boot); + if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT)) pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT); - cpu = get_cpu(); - vcpu_time = &hv_clock[cpu].pvti; - flags = pvclock_read_flags(vcpu_time); - + flags = pvclock_read_flags(&hv_clock_boot[0].pvti); kvm_sched_clock_init(flags & PVCLOCK_TSC_STABLE_BIT); - put_cpu(); x86_platform.calibrate_tsc = kvm_get_tsc_khz; x86_platform.calibrate_cpu = kvm_get_tsc_khz; x86_platform.get_wallclock = kvm_get_wallclock; x86_platform.set_wallclock = kvm_set_wallclock; #ifdef CONFIG_X86_LOCAL_APIC - x86_cpuinit.early_percpu_clock_init = - kvm_setup_secondary_clock; + x86_cpuinit.early_percpu_clock_init = kvm_setup_secondary_clock; #endif x86_platform.save_sched_clock_state = kvm_save_sched_clock_state; x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state; @@ -347,34 +325,3 @@ void __init kvmclock_init(void) clocksource_register_hz(&kvm_clock, NSEC_PER_SEC); pv_info.name = "KVM"; } - -int __init kvm_setup_vsyscall_timeinfo(void) -{ -#ifdef CONFIG_X86_64 - int cpu; - u8 flags; - struct pvclock_vcpu_time_info *vcpu_time; - unsigned int size; - - if (!hv_clock) - return 0; - - size = PAGE_ALIGN(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS); - - cpu = get_cpu(); - - vcpu_time = &hv_clock[cpu].pvti; - flags = pvclock_read_flags(vcpu_time); - - if (!(flags & PVCLOCK_TSC_STABLE_BIT)) { - put_cpu(); - return 1; - } - - pvclock_set_pvti_cpu0_va(hv_clock); - put_cpu(); - - kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; -#endif - return 0; -} diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index c9b14020f4dd..733e6ace0fa4 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -100,6 +100,102 @@ static struct ldt_struct *alloc_ldt_struct(unsigned int num_entries) return new_ldt; } +#ifdef CONFIG_PAGE_TABLE_ISOLATION + +static void do_sanity_check(struct mm_struct *mm, + bool had_kernel_mapping, + bool had_user_mapping) +{ + if (mm->context.ldt) { + /* + * We already had an LDT. The top-level entry should already + * have been allocated and synchronized with the usermode + * tables. + */ + WARN_ON(!had_kernel_mapping); + if (static_cpu_has(X86_FEATURE_PTI)) + WARN_ON(!had_user_mapping); + } else { + /* + * This is the first time we're mapping an LDT for this process. + * Sync the pgd to the usermode tables. + */ + WARN_ON(had_kernel_mapping); + if (static_cpu_has(X86_FEATURE_PTI)) + WARN_ON(had_user_mapping); + } +} + +#ifdef CONFIG_X86_PAE + +static pmd_t *pgd_to_pmd_walk(pgd_t *pgd, unsigned long va) +{ + p4d_t *p4d; + pud_t *pud; + + if (pgd->pgd == 0) + return NULL; + + p4d = p4d_offset(pgd, va); + if (p4d_none(*p4d)) + return NULL; + + pud = pud_offset(p4d, va); + if (pud_none(*pud)) + return NULL; + + return pmd_offset(pud, va); +} + +static void map_ldt_struct_to_user(struct mm_struct *mm) +{ + pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR); + pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); + pmd_t *k_pmd, *u_pmd; + + k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR); + u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR); + + if (static_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt) + set_pmd(u_pmd, *k_pmd); +} + +static void sanity_check_ldt_mapping(struct mm_struct *mm) +{ + pgd_t *k_pgd = pgd_offset(mm, LDT_BASE_ADDR); + pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); + bool had_kernel, had_user; + pmd_t *k_pmd, *u_pmd; + + k_pmd = pgd_to_pmd_walk(k_pgd, LDT_BASE_ADDR); + u_pmd = pgd_to_pmd_walk(u_pgd, LDT_BASE_ADDR); + had_kernel = (k_pmd->pmd != 0); + had_user = (u_pmd->pmd != 0); + + do_sanity_check(mm, had_kernel, had_user); +} + +#else /* !CONFIG_X86_PAE */ + +static void map_ldt_struct_to_user(struct mm_struct *mm) +{ + pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR); + + if (static_cpu_has(X86_FEATURE_PTI) && !mm->context.ldt) + set_pgd(kernel_to_user_pgdp(pgd), *pgd); +} + +static void sanity_check_ldt_mapping(struct mm_struct *mm) +{ + pgd_t *pgd = pgd_offset(mm, LDT_BASE_ADDR); + bool had_kernel = (pgd->pgd != 0); + bool had_user = (kernel_to_user_pgdp(pgd)->pgd != 0); + + do_sanity_check(mm, had_kernel, had_user); +} + +#endif /* CONFIG_X86_PAE */ + /* * If PTI is enabled, this maps the LDT into the kernelmode and * usermode tables for the given mm. @@ -115,9 +211,8 @@ static struct ldt_struct *alloc_ldt_struct(unsigned int num_entries) static int map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) { -#ifdef CONFIG_PAGE_TABLE_ISOLATION - bool is_vmalloc, had_top_level_entry; unsigned long va; + bool is_vmalloc; spinlock_t *ptl; pgd_t *pgd; int i; @@ -131,13 +226,15 @@ map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) */ WARN_ON(ldt->slot != -1); + /* Check if the current mappings are sane */ + sanity_check_ldt_mapping(mm); + /* * Did we already have the top level entry allocated? We can't * use pgd_none() for this because it doens't do anything on * 4-level page table kernels. */ pgd = pgd_offset(mm, LDT_BASE_ADDR); - had_top_level_entry = (pgd->pgd != 0); is_vmalloc = is_vmalloc_addr(ldt->entries); @@ -172,41 +269,31 @@ map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) pte_unmap_unlock(ptep, ptl); } - if (mm->context.ldt) { - /* - * We already had an LDT. The top-level entry should already - * have been allocated and synchronized with the usermode - * tables. - */ - WARN_ON(!had_top_level_entry); - if (static_cpu_has(X86_FEATURE_PTI)) - WARN_ON(!kernel_to_user_pgdp(pgd)->pgd); - } else { - /* - * This is the first time we're mapping an LDT for this process. - * Sync the pgd to the usermode tables. - */ - WARN_ON(had_top_level_entry); - if (static_cpu_has(X86_FEATURE_PTI)) { - WARN_ON(kernel_to_user_pgdp(pgd)->pgd); - set_pgd(kernel_to_user_pgdp(pgd), *pgd); - } - } + /* Propagate LDT mapping to the user page-table */ + map_ldt_struct_to_user(mm); va = (unsigned long)ldt_slot_va(slot); flush_tlb_mm_range(mm, va, va + LDT_SLOT_STRIDE, 0); ldt->slot = slot; -#endif return 0; } +#else /* !CONFIG_PAGE_TABLE_ISOLATION */ + +static int +map_ldt_struct(struct mm_struct *mm, struct ldt_struct *ldt, int slot) +{ + return 0; +} +#endif /* CONFIG_PAGE_TABLE_ISOLATION */ + static void free_ldt_pgtables(struct mm_struct *mm) { #ifdef CONFIG_PAGE_TABLE_ISOLATION struct mmu_gather tlb; unsigned long start = LDT_BASE_ADDR; - unsigned long end = start + (1UL << PGDIR_SHIFT); + unsigned long end = LDT_END_ADDR; if (!static_cpu_has(X86_FEATURE_PTI)) return; diff --git a/arch/x86/kernel/machine_kexec_32.c b/arch/x86/kernel/machine_kexec_32.c index d1ab07ec8c9a..5409c2800ab5 100644 --- a/arch/x86/kernel/machine_kexec_32.c +++ b/arch/x86/kernel/machine_kexec_32.c @@ -56,7 +56,7 @@ static void load_segments(void) static void machine_kexec_free_page_tables(struct kimage *image) { - free_page((unsigned long)image->arch.pgd); + free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER); image->arch.pgd = NULL; #ifdef CONFIG_X86_PAE free_page((unsigned long)image->arch.pmd0); @@ -72,7 +72,8 @@ static void machine_kexec_free_page_tables(struct kimage *image) static int machine_kexec_alloc_page_tables(struct kimage *image) { - image->arch.pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL); + image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + PGD_ALLOCATION_ORDER); #ifdef CONFIG_X86_PAE image->arch.pmd0 = (pmd_t *)get_zeroed_page(GFP_KERNEL); image->arch.pmd1 = (pmd_t *)get_zeroed_page(GFP_KERNEL); diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 99dc79e76bdc..930c88341e4e 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -88,10 +88,12 @@ unsigned paravirt_patch_call(void *insnbuf, struct branch *b = insnbuf; unsigned long delta = (unsigned long)target - (addr+5); - if (tgt_clobbers & ~site_clobbers) - return len; /* target would clobber too much for this site */ - if (len < 5) + if (len < 5) { +#ifdef CONFIG_RETPOLINE + WARN_ONCE("Failing to patch indirect CALL in %ps\n", (void *)addr); +#endif return len; /* call too long for patch site */ + } b->opcode = 0xe8; /* call */ b->delta = delta; @@ -106,8 +108,12 @@ unsigned paravirt_patch_jmp(void *insnbuf, const void *target, struct branch *b = insnbuf; unsigned long delta = (unsigned long)target - (addr+5); - if (len < 5) + if (len < 5) { +#ifdef CONFIG_RETPOLINE + WARN_ONCE("Failing to patch indirect JMP in %ps\n", (void *)addr); +#endif return len; /* call too long for patch site */ + } b->opcode = 0xe9; /* jmp */ b->delta = delta; diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c index 9edadabf04f6..9cb98f7b07c9 100644 --- a/arch/x86/kernel/paravirt_patch_64.c +++ b/arch/x86/kernel/paravirt_patch_64.c @@ -20,7 +20,7 @@ DEF_NATIVE(, mov64, "mov %rdi, %rax"); #if defined(CONFIG_PARAVIRT_SPINLOCKS) DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)"); -DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %rax, %rax"); +DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %eax, %eax"); #endif unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len) diff --git a/arch/x86/kernel/pci-iommu_table.c b/arch/x86/kernel/pci-iommu_table.c index 4dfd90a75e63..2e9006c1e240 100644 --- a/arch/x86/kernel/pci-iommu_table.c +++ b/arch/x86/kernel/pci-iommu_table.c @@ -60,7 +60,7 @@ void __init check_iommu_entries(struct iommu_table_entry *start, printk(KERN_ERR "CYCLIC DEPENDENCY FOUND! %pS depends on %pS and vice-versa. BREAKING IT.\n", p->detect, q->detect); /* Heavy handed way..*/ - x->depend = 0; + x->depend = NULL; } } diff --git a/arch/x86/kernel/pcspeaker.c b/arch/x86/kernel/pcspeaker.c index da5190a1ea16..4a710ffffd9a 100644 --- a/arch/x86/kernel/pcspeaker.c +++ b/arch/x86/kernel/pcspeaker.c @@ -9,6 +9,6 @@ static __init int add_pcspkr(void) pd = platform_device_register_simple("pcspkr", -1, NULL, 0); - return IS_ERR(pd) ? PTR_ERR(pd) : 0; + return PTR_ERR_OR_ZERO(pd); } device_initcall(add_pcspkr); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 30ca2d1a9231..c93fcfdf1673 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -57,14 +57,12 @@ __visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = { */ .sp0 = (1UL << (BITS_PER_LONG-1)) + 1, -#ifdef CONFIG_X86_64 /* * .sp1 is cpu_current_top_of_stack. The init task never * runs user code, but cpu_current_top_of_stack should still * be well defined before the first context switch. */ .sp1 = TOP_OF_INIT_STACK, -#endif #ifdef CONFIG_X86_32 .ss0 = __KERNEL_DS, diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 0ae659de21eb..2924fd447e61 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -285,7 +285,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) * current_thread_info(). Refresh the SYSENTER configuration in * case prev or next is vm86. */ - update_sp0(next_p); + update_task_stack(next_p); refresh_sysenter_cs(next); this_cpu_write(cpu_current_top_of_stack, (unsigned long)task_stack_page(next_p) + diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 12bb445fb98d..476e3ddf8890 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -478,7 +478,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p)); /* Reload sp0. */ - update_sp0(next_p); + update_task_stack(next_p); /* * Now maybe reload the debug registers and handle I/O bitmaps diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 2f86d883dd95..8fe740e22030 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -823,6 +823,12 @@ void __init setup_arch(char **cmdline_p) memblock_reserve(__pa_symbol(_text), (unsigned long)__bss_stop - (unsigned long)_text); + /* + * Make sure page 0 is always reserved because on systems with + * L1TF its contents can be leaked to user processes. + */ + memblock_reserve(0, PAGE_SIZE); + early_reserve_initrd(); /* @@ -866,6 +872,8 @@ void __init setup_arch(char **cmdline_p) idt_setup_early_traps(); early_cpu_init(); + arch_init_ideal_nops(); + jump_label_init(); early_ioremap_init(); setup_olpc_ofw_pgd(); @@ -1012,6 +1020,7 @@ void __init setup_arch(char **cmdline_p) */ init_hypervisor_platform(); + tsc_early_init(); x86_init.resources.probe_roms(); /* after parse_early_param, so could debug it */ @@ -1197,11 +1206,6 @@ void __init setup_arch(char **cmdline_p) memblock_find_dma_reserve(); -#ifdef CONFIG_KVM_GUEST - kvmclock_init(); -#endif - - tsc_early_delay_calibrate(); if (!early_xdbc_setup_hardware()) early_xdbc_register_console(); @@ -1272,8 +1276,6 @@ void __init setup_arch(char **cmdline_p) mcheck_init(); - arch_init_ideal_nops(); - register_refined_jiffies(CLOCK_TICK_RATE); #ifdef CONFIG_EFI diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 5c574dff4c1a..04adc8d60aed 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -261,6 +261,7 @@ __visible void __irq_entry smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); inc_irq_stat(irq_resched_count); + kvm_set_cpu_l1tf_flush_l1d(); if (trace_resched_ipi_enabled()) { /* diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index c2f7d1d2a5c3..f02ecaf97904 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -80,6 +80,7 @@ #include <asm/intel-family.h> #include <asm/cpu_device_id.h> #include <asm/spec-ctrl.h> +#include <asm/hw_irq.h> /* representing HT siblings of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); @@ -221,6 +222,11 @@ static void notrace start_secondary(void *unused) #ifdef CONFIG_X86_32 /* switch away from the initial page table */ load_cr3(swapper_pg_dir); + /* + * Initialize the CR4 shadow before doing anything that could + * try to read it. + */ + cr4_init_shadow(); __flush_tlb_all(); #endif load_current_idt(); @@ -266,6 +272,23 @@ static void notrace start_secondary(void *unused) } /** + * topology_is_primary_thread - Check whether CPU is the primary SMT thread + * @cpu: CPU to check + */ +bool topology_is_primary_thread(unsigned int cpu) +{ + return apic_id_is_primary_thread(per_cpu(x86_cpu_to_apicid, cpu)); +} + +/** + * topology_smt_supported - Check whether SMT is supported by the CPUs + */ +bool topology_smt_supported(void) +{ + return smp_num_siblings > 1; +} + +/** * topology_phys_to_logical_pkg - Map a physical package id to a logical * * Returns logical package id or -1 if not found diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 093f2ea5dd56..7627455047c2 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c @@ -81,16 +81,6 @@ EXPORT_SYMBOL_GPL(save_stack_trace_tsk); #ifdef CONFIG_HAVE_RELIABLE_STACKTRACE -#define STACKTRACE_DUMP_ONCE(task) ({ \ - static bool __section(.data.unlikely) __dumped; \ - \ - if (!__dumped) { \ - __dumped = true; \ - WARN_ON(1); \ - show_stack(task, NULL); \ - } \ -}) - static int __always_inline __save_stack_trace_reliable(struct stack_trace *trace, struct task_struct *task) @@ -99,30 +89,25 @@ __save_stack_trace_reliable(struct stack_trace *trace, struct pt_regs *regs; unsigned long addr; - for (unwind_start(&state, task, NULL, NULL); !unwind_done(&state); + for (unwind_start(&state, task, NULL, NULL); + !unwind_done(&state) && !unwind_error(&state); unwind_next_frame(&state)) { regs = unwind_get_entry_regs(&state, NULL); if (regs) { + /* Success path for user tasks */ + if (user_mode(regs)) + goto success; + /* * Kernel mode registers on the stack indicate an * in-kernel interrupt or exception (e.g., preemption * or a page fault), which can make frame pointers * unreliable. */ - if (!user_mode(regs)) - return -EINVAL; - /* - * The last frame contains the user mode syscall - * pt_regs. Skip it and finish the unwind. - */ - unwind_next_frame(&state); - if (!unwind_done(&state)) { - STACKTRACE_DUMP_ONCE(task); + if (IS_ENABLED(CONFIG_FRAME_POINTER)) return -EINVAL; - } - break; } addr = unwind_get_return_address(&state); @@ -132,21 +117,22 @@ __save_stack_trace_reliable(struct stack_trace *trace, * generated code which __kernel_text_address() doesn't know * about. */ - if (!addr) { - STACKTRACE_DUMP_ONCE(task); + if (!addr) return -EINVAL; - } if (save_stack_address(trace, addr, false)) return -EINVAL; } /* Check for stack corruption */ - if (unwind_error(&state)) { - STACKTRACE_DUMP_ONCE(task); + if (unwind_error(&state)) + return -EINVAL; + + /* Success path for non-user tasks, i.e. kthreads and idle tasks */ + if (!(task->flags & (PF_KTHREAD | PF_IDLE))) return -EINVAL; - } +success: if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = ULONG_MAX; diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 774ebafa97c4..be01328eb755 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -12,6 +12,7 @@ #include <linux/clockchips.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/i8253.h> #include <linux/time.h> #include <linux/export.h> diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 74392d9d51e0..1463468ba9a0 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -33,16 +33,13 @@ EXPORT_SYMBOL(cpu_khz); unsigned int __read_mostly tsc_khz; EXPORT_SYMBOL(tsc_khz); +#define KHZ 1000 + /* * TSC can be unstable due to cpufreq or due to unsynced TSCs */ static int __read_mostly tsc_unstable; -/* native_sched_clock() is called before tsc_init(), so - we must start with the TSC soft disabled to prevent - erroneous rdtsc usage on !boot_cpu_has(X86_FEATURE_TSC) processors */ -static int __read_mostly tsc_disabled = -1; - static DEFINE_STATIC_KEY_FALSE(__use_tsc); int tsc_clocksource_reliable; @@ -106,23 +103,6 @@ void cyc2ns_read_end(void) * -johnstul@us.ibm.com "math is hard, lets go shopping!" */ -static void cyc2ns_data_init(struct cyc2ns_data *data) -{ - data->cyc2ns_mul = 0; - data->cyc2ns_shift = 0; - data->cyc2ns_offset = 0; -} - -static void __init cyc2ns_init(int cpu) -{ - struct cyc2ns *c2n = &per_cpu(cyc2ns, cpu); - - cyc2ns_data_init(&c2n->data[0]); - cyc2ns_data_init(&c2n->data[1]); - - seqcount_init(&c2n->seq); -} - static inline unsigned long long cycles_2_ns(unsigned long long cyc) { struct cyc2ns_data data; @@ -138,18 +118,11 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc) return ns; } -static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now) +static void __set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now) { unsigned long long ns_now; struct cyc2ns_data data; struct cyc2ns *c2n; - unsigned long flags; - - local_irq_save(flags); - sched_clock_idle_sleep_event(); - - if (!khz) - goto done; ns_now = cycles_2_ns(tsc_now); @@ -181,13 +154,56 @@ static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_ c2n->data[0] = data; raw_write_seqcount_latch(&c2n->seq); c2n->data[1] = data; +} + +static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now) +{ + unsigned long flags; + + local_irq_save(flags); + sched_clock_idle_sleep_event(); + + if (khz) + __set_cyc2ns_scale(khz, cpu, tsc_now); -done: sched_clock_idle_wakeup_event(); local_irq_restore(flags); } /* + * Initialize cyc2ns for boot cpu + */ +static void __init cyc2ns_init_boot_cpu(void) +{ + struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns); + + seqcount_init(&c2n->seq); + __set_cyc2ns_scale(tsc_khz, smp_processor_id(), rdtsc()); +} + +/* + * Secondary CPUs do not run through tsc_init(), so set up + * all the scale factors for all CPUs, assuming the same + * speed as the bootup CPU. (cpufreq notifiers will fix this + * up if their speed diverges) + */ +static void __init cyc2ns_init_secondary_cpus(void) +{ + unsigned int cpu, this_cpu = smp_processor_id(); + struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns); + struct cyc2ns_data *data = c2n->data; + + for_each_possible_cpu(cpu) { + if (cpu != this_cpu) { + seqcount_init(&c2n->seq); + c2n = per_cpu_ptr(&cyc2ns, cpu); + c2n->data[0] = data[0]; + c2n->data[1] = data[1]; + } + } +} + +/* * Scheduler clock - returns current time in nanosec units. */ u64 native_sched_clock(void) @@ -248,8 +264,7 @@ EXPORT_SYMBOL_GPL(check_tsc_unstable); #ifdef CONFIG_X86_TSC int __init notsc_setup(char *str) { - pr_warn("Kernel compiled with CONFIG_X86_TSC, cannot disable TSC completely\n"); - tsc_disabled = 1; + mark_tsc_unstable("boot parameter notsc"); return 1; } #else @@ -665,30 +680,17 @@ static unsigned long cpu_khz_from_cpuid(void) return eax_base_mhz * 1000; } -/** - * native_calibrate_cpu - calibrate the cpu on boot +/* + * calibrate cpu using pit, hpet, and ptimer methods. They are available + * later in boot after acpi is initialized. */ -unsigned long native_calibrate_cpu(void) +static unsigned long pit_hpet_ptimer_calibrate_cpu(void) { u64 tsc1, tsc2, delta, ref1, ref2; unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX; - unsigned long flags, latch, ms, fast_calibrate; + unsigned long flags, latch, ms; int hpet = is_hpet_enabled(), i, loopmin; - fast_calibrate = cpu_khz_from_cpuid(); - if (fast_calibrate) - return fast_calibrate; - - fast_calibrate = cpu_khz_from_msr(); - if (fast_calibrate) - return fast_calibrate; - - local_irq_save(flags); - fast_calibrate = quick_pit_calibrate(); - local_irq_restore(flags); - if (fast_calibrate) - return fast_calibrate; - /* * Run 5 calibration loops to get the lowest frequency value * (the best estimate). We use two different calibration modes @@ -831,6 +833,37 @@ unsigned long native_calibrate_cpu(void) return tsc_pit_min; } +/** + * native_calibrate_cpu_early - can calibrate the cpu early in boot + */ +unsigned long native_calibrate_cpu_early(void) +{ + unsigned long flags, fast_calibrate = cpu_khz_from_cpuid(); + + if (!fast_calibrate) + fast_calibrate = cpu_khz_from_msr(); + if (!fast_calibrate) { + local_irq_save(flags); + fast_calibrate = quick_pit_calibrate(); + local_irq_restore(flags); + } + return fast_calibrate; +} + + +/** + * native_calibrate_cpu - calibrate the cpu + */ +static unsigned long native_calibrate_cpu(void) +{ + unsigned long tsc_freq = native_calibrate_cpu_early(); + + if (!tsc_freq) + tsc_freq = pit_hpet_ptimer_calibrate_cpu(); + + return tsc_freq; +} + void recalibrate_cpu_khz(void) { #ifndef CONFIG_SMP @@ -1307,7 +1340,7 @@ unreg: static int __init init_tsc_clocksource(void) { - if (!boot_cpu_has(X86_FEATURE_TSC) || tsc_disabled > 0 || !tsc_khz) + if (!boot_cpu_has(X86_FEATURE_TSC) || !tsc_khz) return 0; if (tsc_unstable) @@ -1341,40 +1374,22 @@ unreg: */ device_initcall(init_tsc_clocksource); -void __init tsc_early_delay_calibrate(void) +static bool __init determine_cpu_tsc_frequencies(bool early) { - unsigned long lpj; - - if (!boot_cpu_has(X86_FEATURE_TSC)) - return; - - cpu_khz = x86_platform.calibrate_cpu(); - tsc_khz = x86_platform.calibrate_tsc(); - - tsc_khz = tsc_khz ? : cpu_khz; - if (!tsc_khz) - return; - - lpj = tsc_khz * 1000; - do_div(lpj, HZ); - loops_per_jiffy = lpj; -} - -void __init tsc_init(void) -{ - u64 lpj, cyc; - int cpu; - - if (!boot_cpu_has(X86_FEATURE_TSC)) { - setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); - return; + /* Make sure that cpu and tsc are not already calibrated */ + WARN_ON(cpu_khz || tsc_khz); + + if (early) { + cpu_khz = x86_platform.calibrate_cpu(); + tsc_khz = x86_platform.calibrate_tsc(); + } else { + /* We should not be here with non-native cpu calibration */ + WARN_ON(x86_platform.calibrate_cpu != native_calibrate_cpu); + cpu_khz = pit_hpet_ptimer_calibrate_cpu(); } - cpu_khz = x86_platform.calibrate_cpu(); - tsc_khz = x86_platform.calibrate_tsc(); - /* - * Trust non-zero tsc_khz as authorative, + * Trust non-zero tsc_khz as authoritative, * and use it to sanity check cpu_khz, * which will be off if system timer is off. */ @@ -1383,52 +1398,78 @@ void __init tsc_init(void) else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz) cpu_khz = tsc_khz; - if (!tsc_khz) { - mark_tsc_unstable("could not calculate TSC khz"); - setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); - return; - } + if (tsc_khz == 0) + return false; pr_info("Detected %lu.%03lu MHz processor\n", - (unsigned long)cpu_khz / 1000, - (unsigned long)cpu_khz % 1000); + (unsigned long)cpu_khz / KHZ, + (unsigned long)cpu_khz % KHZ); if (cpu_khz != tsc_khz) { pr_info("Detected %lu.%03lu MHz TSC", - (unsigned long)tsc_khz / 1000, - (unsigned long)tsc_khz % 1000); + (unsigned long)tsc_khz / KHZ, + (unsigned long)tsc_khz % KHZ); } + return true; +} + +static unsigned long __init get_loops_per_jiffy(void) +{ + unsigned long lpj = tsc_khz * KHZ; + do_div(lpj, HZ); + return lpj; +} + +static void __init tsc_enable_sched_clock(void) +{ /* Sanitize TSC ADJUST before cyc2ns gets initialized */ tsc_store_and_check_tsc_adjust(true); + cyc2ns_init_boot_cpu(); + static_branch_enable(&__use_tsc); +} + +void __init tsc_early_init(void) +{ + if (!boot_cpu_has(X86_FEATURE_TSC)) + return; + if (!determine_cpu_tsc_frequencies(true)) + return; + loops_per_jiffy = get_loops_per_jiffy(); + tsc_enable_sched_clock(); +} + +void __init tsc_init(void) +{ /* - * Secondary CPUs do not run through tsc_init(), so set up - * all the scale factors for all CPUs, assuming the same - * speed as the bootup CPU. (cpufreq notifiers will fix this - * up if their speed diverges) + * native_calibrate_cpu_early can only calibrate using methods that are + * available early in boot. */ - cyc = rdtsc(); - for_each_possible_cpu(cpu) { - cyc2ns_init(cpu); - set_cyc2ns_scale(tsc_khz, cpu, cyc); - } + if (x86_platform.calibrate_cpu == native_calibrate_cpu_early) + x86_platform.calibrate_cpu = native_calibrate_cpu; - if (tsc_disabled > 0) + if (!boot_cpu_has(X86_FEATURE_TSC)) { + setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); return; + } - /* now allow native_sched_clock() to use rdtsc */ + if (!tsc_khz) { + /* We failed to determine frequencies earlier, try again */ + if (!determine_cpu_tsc_frequencies(false)) { + mark_tsc_unstable("could not calculate TSC khz"); + setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER); + return; + } + tsc_enable_sched_clock(); + } - tsc_disabled = 0; - static_branch_enable(&__use_tsc); + cyc2ns_init_secondary_cpus(); if (!no_sched_irq_time) enable_sched_clock_irqtime(); - lpj = ((u64)tsc_khz * 1000); - do_div(lpj, HZ); - lpj_fine = lpj; - + lpj_fine = get_loops_per_jiffy(); use_tsc_delay(); check_system_tsc_reliable(); @@ -1455,7 +1496,7 @@ unsigned long calibrate_delay_is_known(void) int constant_tsc = cpu_has(&cpu_data(cpu), X86_FEATURE_CONSTANT_TSC); const struct cpumask *mask = topology_core_cpumask(cpu); - if (tsc_disabled || !constant_tsc || !mask) + if (!constant_tsc || !mask) return 0; sibling = cpumask_any_but(mask, cpu); diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 19afdbd7d0a7..27ef714d886c 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -1,17 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * tsc_msr.c - TSC frequency enumeration via MSR + * TSC frequency enumeration via MSR * - * Copyright (C) 2013 Intel Corporation + * Copyright (C) 2013, 2018 Intel Corporation * Author: Bin Gao <bin.gao@intel.com> - * - * This file is released under the GPLv2. */ #include <linux/kernel.h> -#include <asm/processor.h> -#include <asm/setup.h> + #include <asm/apic.h> +#include <asm/cpu_device_id.h> +#include <asm/intel-family.h> +#include <asm/msr.h> #include <asm/param.h> +#include <asm/tsc.h> #define MAX_NUM_FREQS 9 @@ -23,44 +25,48 @@ * field msr_plat does. */ struct freq_desc { - u8 x86_family; /* CPU family */ - u8 x86_model; /* model */ u8 msr_plat; /* 1: use MSR_PLATFORM_INFO, 0: MSR_IA32_PERF_STATUS */ u32 freqs[MAX_NUM_FREQS]; }; -static struct freq_desc freq_desc_tables[] = { - /* PNW */ - { 6, 0x27, 0, { 0, 0, 0, 0, 0, 99840, 0, 83200 } }, - /* CLV+ */ - { 6, 0x35, 0, { 0, 133200, 0, 0, 0, 99840, 0, 83200 } }, - /* TNG - Intel Atom processor Z3400 series */ - { 6, 0x4a, 1, { 0, 100000, 133300, 0, 0, 0, 0, 0 } }, - /* VLV2 - Intel Atom processor E3000, Z3600, Z3700 series */ - { 6, 0x37, 1, { 83300, 100000, 133300, 116700, 80000, 0, 0, 0 } }, - /* ANN - Intel Atom processor Z3500 series */ - { 6, 0x5a, 1, { 83300, 100000, 133300, 100000, 0, 0, 0, 0 } }, - /* AMT - Intel Atom processor X7-Z8000 and X5-Z8000 series */ - { 6, 0x4c, 1, { 83300, 100000, 133300, 116700, - 80000, 93300, 90000, 88900, 87500 } }, +/* + * Penwell and Clovertrail use spread spectrum clock, + * so the freq number is not exactly the same as reported + * by MSR based on SDM. + */ +static const struct freq_desc freq_desc_pnw = { + 0, { 0, 0, 0, 0, 0, 99840, 0, 83200 } }; -static int match_cpu(u8 family, u8 model) -{ - int i; +static const struct freq_desc freq_desc_clv = { + 0, { 0, 133200, 0, 0, 0, 99840, 0, 83200 } +}; - for (i = 0; i < ARRAY_SIZE(freq_desc_tables); i++) { - if ((family == freq_desc_tables[i].x86_family) && - (model == freq_desc_tables[i].x86_model)) - return i; - } +static const struct freq_desc freq_desc_byt = { + 1, { 83300, 100000, 133300, 116700, 80000, 0, 0, 0 } +}; - return -1; -} +static const struct freq_desc freq_desc_cht = { + 1, { 83300, 100000, 133300, 116700, 80000, 93300, 90000, 88900, 87500 } +}; -/* Map CPU reference clock freq ID(0-7) to CPU reference clock freq(KHz) */ -#define id_to_freq(cpu_index, freq_id) \ - (freq_desc_tables[cpu_index].freqs[freq_id]) +static const struct freq_desc freq_desc_tng = { + 1, { 0, 100000, 133300, 0, 0, 0, 0, 0 } +}; + +static const struct freq_desc freq_desc_ann = { + 1, { 83300, 100000, 133300, 100000, 0, 0, 0, 0 } +}; + +static const struct x86_cpu_id tsc_msr_cpu_ids[] = { + INTEL_CPU_FAM6(ATOM_PENWELL, freq_desc_pnw), + INTEL_CPU_FAM6(ATOM_CLOVERVIEW, freq_desc_clv), + INTEL_CPU_FAM6(ATOM_SILVERMONT1, freq_desc_byt), + INTEL_CPU_FAM6(ATOM_AIRMONT, freq_desc_cht), + INTEL_CPU_FAM6(ATOM_MERRIFIELD, freq_desc_tng), + INTEL_CPU_FAM6(ATOM_MOOREFIELD, freq_desc_ann), + {} +}; /* * MSR-based CPU/TSC frequency discovery for certain CPUs. @@ -70,18 +76,17 @@ static int match_cpu(u8 family, u8 model) */ unsigned long cpu_khz_from_msr(void) { - u32 lo, hi, ratio, freq_id, freq; + u32 lo, hi, ratio, freq; + const struct freq_desc *freq_desc; + const struct x86_cpu_id *id; unsigned long res; - int cpu_index; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) - return 0; - cpu_index = match_cpu(boot_cpu_data.x86, boot_cpu_data.x86_model); - if (cpu_index < 0) + id = x86_match_cpu(tsc_msr_cpu_ids); + if (!id) return 0; - if (freq_desc_tables[cpu_index].msr_plat) { + freq_desc = (struct freq_desc *)id->driver_data; + if (freq_desc->msr_plat) { rdmsr(MSR_PLATFORM_INFO, lo, hi); ratio = (lo >> 8) & 0xff; } else { @@ -91,8 +96,9 @@ unsigned long cpu_khz_from_msr(void) /* Get FSB FREQ ID */ rdmsr(MSR_FSB_FREQ, lo, hi); - freq_id = lo & 0x7; - freq = id_to_freq(cpu_index, freq_id); + + /* Map CPU reference clock freq ID(0-7) to CPU reference clock freq(KHz) */ + freq = freq_desc->freqs[lo & 0x7]; /* TSC frequency = maximum resolved freq * maximum resolved bus ratio */ res = freq * ratio; diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index feb28fee6cea..26038eacf74a 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -198,7 +198,7 @@ static int orc_sort_cmp(const void *_a, const void *_b) * whitelisted .o files which didn't get objtool generation. */ orc_a = cur_orc_table + (a - cur_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED ? -1 : 1; + return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; } #ifdef CONFIG_MODULES @@ -352,7 +352,7 @@ static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr bool unwind_next_frame(struct unwind_state *state) { - unsigned long ip_p, sp, orig_ip, prev_sp = state->sp; + unsigned long ip_p, sp, orig_ip = state->ip, prev_sp = state->sp; enum stack_type prev_type = state->stack_info.type; struct orc_entry *orc; bool indirect = false; @@ -363,9 +363,9 @@ bool unwind_next_frame(struct unwind_state *state) /* Don't let modules unload while we're reading their ORC data. */ preempt_disable(); - /* Have we reached the end? */ + /* End-of-stack check for user tasks: */ if (state->regs && user_mode(state->regs)) - goto done; + goto the_end; /* * Find the orc_entry associated with the text address. @@ -374,9 +374,16 @@ bool unwind_next_frame(struct unwind_state *state) * calls and calls to noreturn functions. */ orc = orc_find(state->signal ? state->ip : state->ip - 1); - if (!orc || orc->sp_reg == ORC_REG_UNDEFINED) - goto done; - orig_ip = state->ip; + if (!orc) + goto err; + + /* End-of-stack check for kernel threads: */ + if (orc->sp_reg == ORC_REG_UNDEFINED) { + if (!orc->end) + goto err; + + goto the_end; + } /* Find the previous frame's stack: */ switch (orc->sp_reg) { @@ -402,7 +409,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!state->regs || !state->full_regs) { orc_warn("missing regs for base reg R10 at ip %pB\n", (void *)state->ip); - goto done; + goto err; } sp = state->regs->r10; break; @@ -411,7 +418,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!state->regs || !state->full_regs) { orc_warn("missing regs for base reg R13 at ip %pB\n", (void *)state->ip); - goto done; + goto err; } sp = state->regs->r13; break; @@ -420,7 +427,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!state->regs || !state->full_regs) { orc_warn("missing regs for base reg DI at ip %pB\n", (void *)state->ip); - goto done; + goto err; } sp = state->regs->di; break; @@ -429,7 +436,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!state->regs || !state->full_regs) { orc_warn("missing regs for base reg DX at ip %pB\n", (void *)state->ip); - goto done; + goto err; } sp = state->regs->dx; break; @@ -437,12 +444,12 @@ bool unwind_next_frame(struct unwind_state *state) default: orc_warn("unknown SP base reg %d for ip %pB\n", orc->sp_reg, (void *)state->ip); - goto done; + goto err; } if (indirect) { if (!deref_stack_reg(state, sp, &sp)) - goto done; + goto err; } /* Find IP, SP and possibly regs: */ @@ -451,7 +458,7 @@ bool unwind_next_frame(struct unwind_state *state) ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip)) - goto done; + goto err; state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, state->ip, (void *)ip_p); @@ -465,7 +472,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn("can't dereference registers at %p for ip %pB\n", (void *)sp, (void *)orig_ip); - goto done; + goto err; } state->regs = (struct pt_regs *)sp; @@ -477,7 +484,7 @@ bool unwind_next_frame(struct unwind_state *state) if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn("can't dereference iret registers at %p for ip %pB\n", (void *)sp, (void *)orig_ip); - goto done; + goto err; } state->regs = (void *)sp - IRET_FRAME_OFFSET; @@ -500,18 +507,18 @@ bool unwind_next_frame(struct unwind_state *state) case ORC_REG_PREV_SP: if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp)) - goto done; + goto err; break; case ORC_REG_BP: if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp)) - goto done; + goto err; break; default: orc_warn("unknown BP base reg %d for ip %pB\n", orc->bp_reg, (void *)orig_ip); - goto done; + goto err; } /* Prevent a recursive loop due to bad ORC data: */ @@ -520,13 +527,16 @@ bool unwind_next_frame(struct unwind_state *state) state->sp <= prev_sp) { orc_warn("stack going in the wrong direction? ip=%pB\n", (void *)orig_ip); - goto done; + goto err; } preempt_enable(); return true; -done: +err: + state->error = true; + +the_end: preempt_enable(); state->stack_info.type = STACK_TYPE_UNKNOWN; return false; diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c index 9d0b5af7db91..1c03e4aa6474 100644 --- a/arch/x86/kernel/vm86_32.c +++ b/arch/x86/kernel/vm86_32.c @@ -149,7 +149,7 @@ void save_v86_state(struct kernel_vm86_regs *regs, int retval) preempt_disable(); tsk->thread.sp0 = vm86->saved_sp0; tsk->thread.sysenter_cs = __KERNEL_CS; - update_sp0(tsk); + update_task_stack(tsk); refresh_sysenter_cs(&tsk->thread); vm86->saved_sp0 = 0; preempt_enable(); @@ -374,7 +374,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus) refresh_sysenter_cs(&tsk->thread); } - update_sp0(tsk); + update_task_stack(tsk); preempt_enable(); if (vm86->flags & VM86_SCREEN_BITMAP) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 5e1458f609a1..8bde0a419f86 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -55,19 +55,22 @@ jiffies_64 = jiffies; * so we can enable protection checks as well as retain 2MB large page * mappings for kernel text. */ -#define X64_ALIGN_RODATA_BEGIN . = ALIGN(HPAGE_SIZE); +#define X86_ALIGN_RODATA_BEGIN . = ALIGN(HPAGE_SIZE); -#define X64_ALIGN_RODATA_END \ +#define X86_ALIGN_RODATA_END \ . = ALIGN(HPAGE_SIZE); \ - __end_rodata_hpage_align = .; + __end_rodata_hpage_align = .; \ + __end_rodata_aligned = .; #define ALIGN_ENTRY_TEXT_BEGIN . = ALIGN(PMD_SIZE); #define ALIGN_ENTRY_TEXT_END . = ALIGN(PMD_SIZE); #else -#define X64_ALIGN_RODATA_BEGIN -#define X64_ALIGN_RODATA_END +#define X86_ALIGN_RODATA_BEGIN +#define X86_ALIGN_RODATA_END \ + . = ALIGN(PAGE_SIZE); \ + __end_rodata_aligned = .; #define ALIGN_ENTRY_TEXT_BEGIN #define ALIGN_ENTRY_TEXT_END @@ -141,9 +144,9 @@ SECTIONS /* .text should occupy whole number of pages */ . = ALIGN(PAGE_SIZE); - X64_ALIGN_RODATA_BEGIN + X86_ALIGN_RODATA_BEGIN RO_DATA(PAGE_SIZE) - X64_ALIGN_RODATA_END + X86_ALIGN_RODATA_END /* Data */ .data : AT(ADDR(.data) - LOAD_OFFSET) { diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 3ab867603e81..2792b5573818 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -109,7 +109,7 @@ struct x86_cpuinit_ops x86_cpuinit = { static void default_nmi_init(void) { }; struct x86_platform_ops x86_platform __ro_after_init = { - .calibrate_cpu = native_calibrate_cpu, + .calibrate_cpu = native_calibrate_cpu_early, .calibrate_tsc = native_calibrate_tsc, .get_wallclock = mach_get_cmos_time, .set_wallclock = mach_set_rtc_mmss, diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 92fd433c50b9..1bbec387d289 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -85,7 +85,7 @@ config KVM_AMD_SEV def_bool y bool "AMD Secure Encrypted Virtualization (SEV) support" depends on KVM_AMD && X86_64 - depends on CRYPTO_DEV_CCP && CRYPTO_DEV_CCP_DD && CRYPTO_DEV_SP_PSP + depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) ---help--- Provides support for launching Encrypted VMs on AMD processors. diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index b5cd8465d44f..d536d457517b 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1379,7 +1379,7 @@ static void apic_timer_expired(struct kvm_lapic *apic) * using swait_active() is safe. */ if (swait_active(q)) - swake_up(q); + swake_up_one(q); if (apic_lvtt_tscdeadline(apic)) ktimer->expired_tscdeadline = ktimer->tscdeadline; diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index d594690d8b95..a44e568363a4 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -890,7 +890,7 @@ static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache, if (cache->nobjs >= min) return 0; while (cache->nobjs < ARRAY_SIZE(cache->objects)) { - page = (void *)__get_free_page(GFP_KERNEL); + page = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); if (!page) return -ENOMEM; cache->objects[cache->nobjs++] = page; @@ -3840,6 +3840,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, { int r = 1; + vcpu->arch.l1tf_flush_l1d = true; switch (vcpu->arch.apf.host_apf_reason) { default: trace_kvm_page_fault(fault_address, error_code); diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 1689f433f3a0..46b428c0990e 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -188,6 +188,150 @@ module_param(ple_window_max, uint, 0444); extern const ulong vmx_return; +static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); +static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); +static DEFINE_MUTEX(vmx_l1d_flush_mutex); + +/* Storage for pre module init parameter parsing */ +static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; + +static const struct { + const char *option; + enum vmx_l1d_flush_state cmd; +} vmentry_l1d_param[] = { + {"auto", VMENTER_L1D_FLUSH_AUTO}, + {"never", VMENTER_L1D_FLUSH_NEVER}, + {"cond", VMENTER_L1D_FLUSH_COND}, + {"always", VMENTER_L1D_FLUSH_ALWAYS}, +}; + +#define L1D_CACHE_ORDER 4 +static void *vmx_l1d_flush_pages; + +static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) +{ + struct page *page; + unsigned int i; + + if (!enable_ept) { + l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; + return 0; + } + + if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { + u64 msr; + + rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr); + if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { + l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; + return 0; + } + } + + /* If set to auto use the default l1tf mitigation method */ + if (l1tf == VMENTER_L1D_FLUSH_AUTO) { + switch (l1tf_mitigation) { + case L1TF_MITIGATION_OFF: + l1tf = VMENTER_L1D_FLUSH_NEVER; + break; + case L1TF_MITIGATION_FLUSH_NOWARN: + case L1TF_MITIGATION_FLUSH: + case L1TF_MITIGATION_FLUSH_NOSMT: + l1tf = VMENTER_L1D_FLUSH_COND; + break; + case L1TF_MITIGATION_FULL: + case L1TF_MITIGATION_FULL_FORCE: + l1tf = VMENTER_L1D_FLUSH_ALWAYS; + break; + } + } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { + l1tf = VMENTER_L1D_FLUSH_ALWAYS; + } + + if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && + !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { + page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); + if (!page) + return -ENOMEM; + vmx_l1d_flush_pages = page_address(page); + + /* + * Initialize each page with a different pattern in + * order to protect against KSM in the nested + * virtualization case. + */ + for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { + memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, + PAGE_SIZE); + } + } + + l1tf_vmx_mitigation = l1tf; + + if (l1tf != VMENTER_L1D_FLUSH_NEVER) + static_branch_enable(&vmx_l1d_should_flush); + else + static_branch_disable(&vmx_l1d_should_flush); + + if (l1tf == VMENTER_L1D_FLUSH_COND) + static_branch_enable(&vmx_l1d_flush_cond); + else + static_branch_disable(&vmx_l1d_flush_cond); + return 0; +} + +static int vmentry_l1d_flush_parse(const char *s) +{ + unsigned int i; + + if (s) { + for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { + if (sysfs_streq(s, vmentry_l1d_param[i].option)) + return vmentry_l1d_param[i].cmd; + } + } + return -EINVAL; +} + +static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) +{ + int l1tf, ret; + + if (!boot_cpu_has(X86_BUG_L1TF)) + return 0; + + l1tf = vmentry_l1d_flush_parse(s); + if (l1tf < 0) + return l1tf; + + /* + * Has vmx_init() run already? If not then this is the pre init + * parameter parsing. In that case just store the value and let + * vmx_init() do the proper setup after enable_ept has been + * established. + */ + if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { + vmentry_l1d_flush_param = l1tf; + return 0; + } + + mutex_lock(&vmx_l1d_flush_mutex); + ret = vmx_setup_l1d_flush(l1tf); + mutex_unlock(&vmx_l1d_flush_mutex); + return ret; +} + +static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) +{ + return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); +} + +static const struct kernel_param_ops vmentry_l1d_flush_ops = { + .set = vmentry_l1d_flush_set, + .get = vmentry_l1d_flush_get, +}; +module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); + struct kvm_vmx { struct kvm kvm; @@ -757,6 +901,11 @@ static inline int pi_test_sn(struct pi_desc *pi_desc) (unsigned long *)&pi_desc->control); } +struct vmx_msrs { + unsigned int nr; + struct vmx_msr_entry val[NR_AUTOLOAD_MSRS]; +}; + struct vcpu_vmx { struct kvm_vcpu vcpu; unsigned long host_rsp; @@ -790,9 +939,8 @@ struct vcpu_vmx { struct loaded_vmcs *loaded_vmcs; bool __launched; /* temporary, used in vmx_vcpu_run */ struct msr_autoload { - unsigned nr; - struct vmx_msr_entry guest[NR_AUTOLOAD_MSRS]; - struct vmx_msr_entry host[NR_AUTOLOAD_MSRS]; + struct vmx_msrs guest; + struct vmx_msrs host; } msr_autoload; struct { int loaded; @@ -2377,9 +2525,20 @@ static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, vm_exit_controls_clearbit(vmx, exit); } +static int find_msr(struct vmx_msrs *m, unsigned int msr) +{ + unsigned int i; + + for (i = 0; i < m->nr; ++i) { + if (m->val[i].index == msr) + return i; + } + return -ENOENT; +} + static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) { - unsigned i; + int i; struct msr_autoload *m = &vmx->msr_autoload; switch (msr) { @@ -2400,18 +2559,21 @@ static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) } break; } + i = find_msr(&m->guest, msr); + if (i < 0) + goto skip_guest; + --m->guest.nr; + m->guest.val[i] = m->guest.val[m->guest.nr]; + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); - for (i = 0; i < m->nr; ++i) - if (m->guest[i].index == msr) - break; - - if (i == m->nr) +skip_guest: + i = find_msr(&m->host, msr); + if (i < 0) return; - --m->nr; - m->guest[i] = m->guest[m->nr]; - m->host[i] = m->host[m->nr]; - vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr); - vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr); + + --m->host.nr; + m->host.val[i] = m->host.val[m->host.nr]; + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); } static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, @@ -2426,9 +2588,9 @@ static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, } static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, - u64 guest_val, u64 host_val) + u64 guest_val, u64 host_val, bool entry_only) { - unsigned i; + int i, j = 0; struct msr_autoload *m = &vmx->msr_autoload; switch (msr) { @@ -2463,24 +2625,31 @@ static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, wrmsrl(MSR_IA32_PEBS_ENABLE, 0); } - for (i = 0; i < m->nr; ++i) - if (m->guest[i].index == msr) - break; + i = find_msr(&m->guest, msr); + if (!entry_only) + j = find_msr(&m->host, msr); - if (i == NR_AUTOLOAD_MSRS) { + if (i == NR_AUTOLOAD_MSRS || j == NR_AUTOLOAD_MSRS) { printk_once(KERN_WARNING "Not enough msr switch entries. " "Can't add msr %x\n", msr); return; - } else if (i == m->nr) { - ++m->nr; - vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->nr); - vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->nr); } + if (i < 0) { + i = m->guest.nr++; + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); + } + m->guest.val[i].index = msr; + m->guest.val[i].value = guest_val; - m->guest[i].index = msr; - m->guest[i].value = guest_val; - m->host[i].index = msr; - m->host[i].value = host_val; + if (entry_only) + return; + + if (j < 0) { + j = m->host.nr++; + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); + } + m->host.val[j].index = msr; + m->host.val[j].value = host_val; } static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset) @@ -2524,7 +2693,7 @@ static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset) guest_efer &= ~EFER_LME; if (guest_efer != host_efer) add_atomic_switch_msr(vmx, MSR_EFER, - guest_efer, host_efer); + guest_efer, host_efer, false); return false; } else { guest_efer &= ~ignore_bits; @@ -2571,6 +2740,7 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); #ifdef CONFIG_X86_64 int cpu = raw_smp_processor_id(); + unsigned long fs_base, kernel_gs_base; #endif int i; @@ -2586,12 +2756,20 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel; #ifdef CONFIG_X86_64 - save_fsgs_for_kvm(); - vmx->host_state.fs_sel = current->thread.fsindex; - vmx->host_state.gs_sel = current->thread.gsindex; -#else - savesegment(fs, vmx->host_state.fs_sel); - savesegment(gs, vmx->host_state.gs_sel); + if (likely(is_64bit_mm(current->mm))) { + save_fsgs_for_kvm(); + vmx->host_state.fs_sel = current->thread.fsindex; + vmx->host_state.gs_sel = current->thread.gsindex; + fs_base = current->thread.fsbase; + kernel_gs_base = current->thread.gsbase; + } else { +#endif + savesegment(fs, vmx->host_state.fs_sel); + savesegment(gs, vmx->host_state.gs_sel); +#ifdef CONFIG_X86_64 + fs_base = read_msr(MSR_FS_BASE); + kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); + } #endif if (!(vmx->host_state.fs_sel & 7)) { vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel); @@ -2611,10 +2789,10 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) savesegment(ds, vmx->host_state.ds_sel); savesegment(es, vmx->host_state.es_sel); - vmcs_writel(HOST_FS_BASE, current->thread.fsbase); + vmcs_writel(HOST_FS_BASE, fs_base); vmcs_writel(HOST_GS_BASE, cpu_kernelmode_gs_base(cpu)); - vmx->msr_host_kernel_gs_base = current->thread.gsbase; + vmx->msr_host_kernel_gs_base = kernel_gs_base; if (is_long_mode(&vmx->vcpu)) wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); #else @@ -3978,7 +4156,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vcpu->arch.ia32_xss = data; if (vcpu->arch.ia32_xss != host_xss) add_atomic_switch_msr(vmx, MSR_IA32_XSS, - vcpu->arch.ia32_xss, host_xss); + vcpu->arch.ia32_xss, host_xss, false); else clear_atomic_switch_msr(vmx, MSR_IA32_XSS); break; @@ -4322,11 +4500,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) vmcs_conf->order = get_order(vmcs_conf->size); vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; - /* KVM supports Enlightened VMCS v1 only */ - if (static_branch_unlikely(&enable_evmcs)) - vmcs_conf->revision_id = KVM_EVMCS_VERSION; - else - vmcs_conf->revision_id = vmx_msr_low; + vmcs_conf->revision_id = vmx_msr_low; vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; @@ -4396,7 +4570,13 @@ static struct vmcs *alloc_vmcs_cpu(int cpu) return NULL; vmcs = page_address(pages); memset(vmcs, 0, vmcs_config.size); - vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */ + + /* KVM supports Enlightened VMCS v1 only */ + if (static_branch_unlikely(&enable_evmcs)) + vmcs->revision_id = KVM_EVMCS_VERSION; + else + vmcs->revision_id = vmcs_config.revision_id; + return vmcs; } @@ -4564,6 +4744,19 @@ static __init int alloc_kvm_area(void) return -ENOMEM; } + /* + * When eVMCS is enabled, alloc_vmcs_cpu() sets + * vmcs->revision_id to KVM_EVMCS_VERSION instead of + * revision_id reported by MSR_IA32_VMX_BASIC. + * + * However, even though not explictly documented by + * TLFS, VMXArea passed as VMXON argument should + * still be marked with revision_id reported by + * physical CPU. + */ + if (static_branch_unlikely(&enable_evmcs)) + vmcs->revision_id = vmcs_config.revision_id; + per_cpu(vmxarea, cpu) = vmcs; } return 0; @@ -6250,9 +6443,9 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); - vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host)); + vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); - vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest)); + vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); @@ -6272,8 +6465,7 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx) ++vmx->nmsrs; } - if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) - rdmsrl(MSR_IA32_ARCH_CAPABILITIES, vmx->arch_capabilities); + vmx->arch_capabilities = kvm_get_arch_capabilities(); vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl); @@ -7869,6 +8061,8 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) HRTIMER_MODE_REL_PINNED); vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; + vmx->nested.vpid02 = allocate_vpid(); + vmx->nested.vmxon = true; return 0; @@ -8456,21 +8650,20 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); - gva_t vmcs_gva; + unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); + u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); + gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; struct x86_exception e; + gva_t gva; if (!nested_vmx_check_permission(vcpu)) return 1; - if (get_vmx_mem_address(vcpu, exit_qualification, - vmx_instruction_info, true, &vmcs_gva)) + if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva)) return 1; /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ - if (kvm_write_guest_virt_system(vcpu, vmcs_gva, - (void *)&to_vmx(vcpu)->nested.current_vmptr, - sizeof(u64), &e)) { + if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, + sizeof(gpa_t), &e)) { kvm_inject_page_fault(vcpu, &e); return 1; } @@ -9523,6 +9716,79 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) } } +/* + * Software based L1D cache flush which is used when microcode providing + * the cache control MSR is not loaded. + * + * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to + * flush it is required to read in 64 KiB because the replacement algorithm + * is not exactly LRU. This could be sized at runtime via topology + * information but as all relevant affected CPUs have 32KiB L1D cache size + * there is no point in doing so. + */ +#define L1D_CACHE_ORDER 4 +static void *vmx_l1d_flush_pages; + +static void vmx_l1d_flush(struct kvm_vcpu *vcpu) +{ + int size = PAGE_SIZE << L1D_CACHE_ORDER; + + /* + * This code is only executed when the the flush mode is 'cond' or + * 'always' + */ + if (static_branch_likely(&vmx_l1d_flush_cond)) { + bool flush_l1d; + + /* + * Clear the per-vcpu flush bit, it gets set again + * either from vcpu_run() or from one of the unsafe + * VMEXIT handlers. + */ + flush_l1d = vcpu->arch.l1tf_flush_l1d; + vcpu->arch.l1tf_flush_l1d = false; + + /* + * Clear the per-cpu flush bit, it gets set again from + * the interrupt handlers. + */ + flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); + kvm_clear_cpu_l1tf_flush_l1d(); + + if (!flush_l1d) + return; + } + + vcpu->stat.l1d_flush++; + + if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { + wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); + return; + } + + asm volatile( + /* First ensure the pages are in the TLB */ + "xorl %%eax, %%eax\n" + ".Lpopulate_tlb:\n\t" + "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" + "addl $4096, %%eax\n\t" + "cmpl %%eax, %[size]\n\t" + "jne .Lpopulate_tlb\n\t" + "xorl %%eax, %%eax\n\t" + "cpuid\n\t" + /* Now fill the cache */ + "xorl %%eax, %%eax\n" + ".Lfill_cache:\n" + "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" + "addl $64, %%eax\n\t" + "cmpl %%eax, %[size]\n\t" + "jne .Lfill_cache\n\t" + "lfence\n" + :: [flush_pages] "r" (vmx_l1d_flush_pages), + [size] "r" (size) + : "eax", "ebx", "ecx", "edx"); +} + static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); @@ -9924,7 +10190,7 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) clear_atomic_switch_msr(vmx, msrs[i].msr); else add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, - msrs[i].host); + msrs[i].host, false); } static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu) @@ -10019,6 +10285,9 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) evmcs_rsp = static_branch_unlikely(&enable_evmcs) ? (unsigned long)¤t_evmcs->host_rsp : 0; + if (static_branch_unlikely(&vmx_l1d_should_flush)) + vmx_l1d_flush(vcpu); + asm( /* Store host registers */ "push %%" _ASM_DX "; push %%" _ASM_BP ";" @@ -10346,11 +10615,9 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) goto free_vmcs; } - if (nested) { + if (nested) nested_vmx_setup_ctls_msrs(&vmx->nested.msrs, kvm_vcpu_apicv_active(&vmx->vcpu)); - vmx->nested.vpid02 = allocate_vpid(); - } vmx->nested.posted_intr_nv = -1; vmx->nested.current_vmptr = -1ull; @@ -10367,7 +10634,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) return &vmx->vcpu; free_vmcs: - free_vpid(vmx->nested.vpid02); free_loaded_vmcs(vmx->loaded_vmcs); free_msrs: kfree(vmx->guest_msrs); @@ -10381,10 +10647,37 @@ free_vcpu: return ERR_PTR(err); } +#define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html for details.\n" +#define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html for details.\n" + static int vmx_vm_init(struct kvm *kvm) { if (!ple_gap) kvm->arch.pause_in_guest = true; + + if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { + switch (l1tf_mitigation) { + case L1TF_MITIGATION_OFF: + case L1TF_MITIGATION_FLUSH_NOWARN: + /* 'I explicitly don't care' is set */ + break; + case L1TF_MITIGATION_FLUSH: + case L1TF_MITIGATION_FLUSH_NOSMT: + case L1TF_MITIGATION_FULL: + /* + * Warn upon starting the first VM in a potentially + * insecure environment. + */ + if (cpu_smt_control == CPU_SMT_ENABLED) + pr_warn_once(L1TF_MSG_SMT); + if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) + pr_warn_once(L1TF_MSG_L1D); + break; + case L1TF_MITIGATION_FULL_FORCE: + /* Flush is enforced */ + break; + } + } return 0; } @@ -11238,10 +11531,10 @@ static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) * Set the MSR load/store lists to match L0's settings. */ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); - vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr); - vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host)); - vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr); - vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest)); + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); + vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); + vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); set_cr4_guest_host_mask(vmx); @@ -11753,7 +12046,6 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - u32 msr_entry_idx; u32 exit_qual; int r; @@ -11775,10 +12067,10 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu) nested_get_vmcs12_pages(vcpu, vmcs12); r = EXIT_REASON_MSR_LOAD_FAIL; - msr_entry_idx = nested_vmx_load_msr(vcpu, - vmcs12->vm_entry_msr_load_addr, - vmcs12->vm_entry_msr_load_count); - if (msr_entry_idx) + exit_qual = nested_vmx_load_msr(vcpu, + vmcs12->vm_entry_msr_load_addr, + vmcs12->vm_entry_msr_load_count); + if (exit_qual) goto fail; /* @@ -11878,6 +12170,9 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) return ret; } + /* Hide L1D cache contents from the nested guest. */ + vmx->vcpu.arch.l1tf_flush_l1d = true; + /* * If we're entering a halted L2 vcpu and the L2 vcpu won't be woken * by event injection, halt vcpu. @@ -12398,8 +12693,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vmx_segment_cache_clear(vmx); /* Update any VMCS fields that might have changed while L2 ran */ - vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr); - vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr); + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); if (vmx->hv_deadline_tsc == -1) vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL, @@ -13116,6 +13411,51 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { .enable_smi_window = enable_smi_window, }; +static void vmx_cleanup_l1d_flush(void) +{ + if (vmx_l1d_flush_pages) { + free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); + vmx_l1d_flush_pages = NULL; + } + /* Restore state so sysfs ignores VMX */ + l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; +} + +static void vmx_exit(void) +{ +#ifdef CONFIG_KEXEC_CORE + RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); + synchronize_rcu(); +#endif + + kvm_exit(); + +#if IS_ENABLED(CONFIG_HYPERV) + if (static_branch_unlikely(&enable_evmcs)) { + int cpu; + struct hv_vp_assist_page *vp_ap; + /* + * Reset everything to support using non-enlightened VMCS + * access later (e.g. when we reload the module with + * enlightened_vmcs=0) + */ + for_each_online_cpu(cpu) { + vp_ap = hv_get_vp_assist_page(cpu); + + if (!vp_ap) + continue; + + vp_ap->current_nested_vmcs = 0; + vp_ap->enlighten_vmentry = 0; + } + + static_branch_disable(&enable_evmcs); + } +#endif + vmx_cleanup_l1d_flush(); +} +module_exit(vmx_exit); + static int __init vmx_init(void) { int r; @@ -13150,10 +13490,25 @@ static int __init vmx_init(void) #endif r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), - __alignof__(struct vcpu_vmx), THIS_MODULE); + __alignof__(struct vcpu_vmx), THIS_MODULE); if (r) return r; + /* + * Must be called after kvm_init() so enable_ept is properly set + * up. Hand the parameter mitigation value in which was stored in + * the pre module init parser. If no parameter was given, it will + * contain 'auto' which will be turned into the default 'cond' + * mitigation mode. + */ + if (boot_cpu_has(X86_BUG_L1TF)) { + r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); + if (r) { + vmx_exit(); + return r; + } + } + #ifdef CONFIG_KEXEC_CORE rcu_assign_pointer(crash_vmclear_loaded_vmcss, crash_vmclear_local_loaded_vmcss); @@ -13162,39 +13517,4 @@ static int __init vmx_init(void) return 0; } - -static void __exit vmx_exit(void) -{ -#ifdef CONFIG_KEXEC_CORE - RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); - synchronize_rcu(); -#endif - - kvm_exit(); - -#if IS_ENABLED(CONFIG_HYPERV) - if (static_branch_unlikely(&enable_evmcs)) { - int cpu; - struct hv_vp_assist_page *vp_ap; - /* - * Reset everything to support using non-enlightened VMCS - * access later (e.g. when we reload the module with - * enlightened_vmcs=0) - */ - for_each_online_cpu(cpu) { - vp_ap = hv_get_vp_assist_page(cpu); - - if (!vp_ap) - continue; - - vp_ap->current_nested_vmcs = 0; - vp_ap->enlighten_vmentry = 0; - } - - static_branch_disable(&enable_evmcs); - } -#endif -} - -module_init(vmx_init) -module_exit(vmx_exit) +module_init(vmx_init); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0046aa70205a..a5caa5e5480c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -195,6 +195,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "irq_injections", VCPU_STAT(irq_injections) }, { "nmi_injections", VCPU_STAT(nmi_injections) }, { "req_event", VCPU_STAT(req_event) }, + { "l1d_flush", VCPU_STAT(l1d_flush) }, { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, { "mmu_pte_write", VM_STAT(mmu_pte_write) }, { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, @@ -1097,15 +1098,41 @@ static u32 msr_based_features[] = { MSR_F10H_DECFG, MSR_IA32_UCODE_REV, + MSR_IA32_ARCH_CAPABILITIES, }; static unsigned int num_msr_based_features; +u64 kvm_get_arch_capabilities(void) +{ + u64 data; + + rdmsrl_safe(MSR_IA32_ARCH_CAPABILITIES, &data); + + /* + * If we're doing cache flushes (either "always" or "cond") + * we will do one whenever the guest does a vmlaunch/vmresume. + * If an outer hypervisor is doing the cache flush for us + * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that + * capability to the guest too, and if EPT is disabled we're not + * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will + * require a nested hypervisor to do a flush of its own. + */ + if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER) + data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH; + + return data; +} +EXPORT_SYMBOL_GPL(kvm_get_arch_capabilities); + static int kvm_get_msr_feature(struct kvm_msr_entry *msr) { switch (msr->index) { + case MSR_IA32_ARCH_CAPABILITIES: + msr->data = kvm_get_arch_capabilities(); + break; case MSR_IA32_UCODE_REV: - rdmsrl(msr->index, msr->data); + rdmsrl_safe(msr->index, &msr->data); break; default: if (kvm_x86_ops->get_msr_feature(msr)) @@ -4874,6 +4901,9 @@ static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *v int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, unsigned int bytes, struct x86_exception *exception) { + /* kvm_write_guest_virt_system can pull in tons of pages. */ + vcpu->arch.l1tf_flush_l1d = true; + return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, PFERR_WRITE_MASK, exception); } @@ -6050,6 +6080,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, bool writeback = true; bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable; + vcpu->arch.l1tf_flush_l1d = true; + /* * Clear write_fault_to_shadow_pgtable here to ensure it is * never reused. @@ -7579,6 +7611,7 @@ static int vcpu_run(struct kvm_vcpu *vcpu) struct kvm *kvm = vcpu->kvm; vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); + vcpu->arch.l1tf_flush_l1d = true; for (;;) { if (kvm_vcpu_running(vcpu)) { @@ -8698,6 +8731,7 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) { + vcpu->arch.l1tf_flush_l1d = true; kvm_x86_ops->sched_in(vcpu, cpu); } diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index 298ef1479240..3b24dc05251c 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -256,7 +256,7 @@ ENTRY(__memcpy_mcsafe) /* Copy successful. Return zero */ .L_done_memcpy_trap: - xorq %rax, %rax + xorl %eax, %eax ret ENDPROC(__memcpy_mcsafe) EXPORT_SYMBOL_GPL(__memcpy_mcsafe) diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index 2f3c9196b834..a12afff146d1 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c @@ -111,6 +111,8 @@ static struct addr_marker address_markers[] = { [END_OF_SPACE_NR] = { -1, NULL } }; +#define INIT_PGD ((pgd_t *) &init_top_pgt) + #else /* CONFIG_X86_64 */ enum address_markers_idx { @@ -121,6 +123,9 @@ enum address_markers_idx { #ifdef CONFIG_HIGHMEM PKMAP_BASE_NR, #endif +#ifdef CONFIG_MODIFY_LDT_SYSCALL + LDT_NR, +#endif CPU_ENTRY_AREA_NR, FIXADDR_START_NR, END_OF_SPACE_NR, @@ -134,11 +139,16 @@ static struct addr_marker address_markers[] = { #ifdef CONFIG_HIGHMEM [PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" }, #endif +#ifdef CONFIG_MODIFY_LDT_SYSCALL + [LDT_NR] = { 0UL, "LDT remap" }, +#endif [CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" }, [FIXADDR_START_NR] = { 0UL, "Fixmap area" }, [END_OF_SPACE_NR] = { -1, NULL } }; +#define INIT_PGD (swapper_pg_dir) + #endif /* !CONFIG_X86_64 */ /* Multipliers for offsets within the PTEs */ @@ -496,11 +506,7 @@ static inline bool is_hypervisor_range(int idx) static void ptdump_walk_pgd_level_core(struct seq_file *m, pgd_t *pgd, bool checkwx, bool dmesg) { -#ifdef CONFIG_X86_64 - pgd_t *start = (pgd_t *) &init_top_pgt; -#else - pgd_t *start = swapper_pg_dir; -#endif + pgd_t *start = INIT_PGD; pgprotval_t prot, eff; int i; struct pg_state st = {}; @@ -563,12 +569,13 @@ void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user) } EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs); -static void ptdump_walk_user_pgd_level_checkwx(void) +void ptdump_walk_user_pgd_level_checkwx(void) { #ifdef CONFIG_PAGE_TABLE_ISOLATION - pgd_t *pgd = (pgd_t *) &init_top_pgt; + pgd_t *pgd = INIT_PGD; - if (!static_cpu_has(X86_FEATURE_PTI)) + if (!(__supported_pte_mask & _PAGE_NX) || + !static_cpu_has(X86_FEATURE_PTI)) return; pr_info("x86/mm: Checking user space page tables\n"); @@ -580,7 +587,6 @@ static void ptdump_walk_user_pgd_level_checkwx(void) void ptdump_walk_pgd_level_checkwx(void) { ptdump_walk_pgd_level_core(NULL, NULL, true, false); - ptdump_walk_user_pgd_level_checkwx(); } static int __init pt_dump_init(void) @@ -609,6 +615,9 @@ static int __init pt_dump_init(void) # endif address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE; +# ifdef CONFIG_MODIFY_LDT_SYSCALL + address_markers[LDT_NR].start_address = LDT_BASE_ADDR; +# endif #endif return 0; } diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 2aafa6ab6103..db1c042e9853 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -317,8 +317,6 @@ static noinline int vmalloc_fault(unsigned long address) if (!(address >= VMALLOC_START && address < VMALLOC_END)) return -1; - WARN_ON_ONCE(in_nmi()); - /* * Synchronize this task's top level page-table * with the 'reference' page table. diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index cee58a972cb2..156ed8154af8 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -4,6 +4,8 @@ #include <linux/swap.h> #include <linux/memblock.h> #include <linux/bootmem.h> /* for max_low_pfn */ +#include <linux/swapfile.h> +#include <linux/swapops.h> #include <asm/set_memory.h> #include <asm/e820/api.h> @@ -773,13 +775,44 @@ void free_init_pages(char *what, unsigned long begin, unsigned long end) } } +/* + * begin/end can be in the direct map or the "high kernel mapping" + * used for the kernel image only. free_init_pages() will do the + * right thing for either kind of address. + */ +void free_kernel_image_pages(void *begin, void *end) +{ + unsigned long begin_ul = (unsigned long)begin; + unsigned long end_ul = (unsigned long)end; + unsigned long len_pages = (end_ul - begin_ul) >> PAGE_SHIFT; + + + free_init_pages("unused kernel image", begin_ul, end_ul); + + /* + * PTI maps some of the kernel into userspace. For performance, + * this includes some kernel areas that do not contain secrets. + * Those areas might be adjacent to the parts of the kernel image + * being freed, which may contain secrets. Remove the "high kernel + * image mapping" for these freed areas, ensuring they are not even + * potentially vulnerable to Meltdown regardless of the specific + * optimizations PTI is currently using. + * + * The "noalias" prevents unmapping the direct map alias which is + * needed to access the freed pages. + * + * This is only valid for 64bit kernels. 32bit has only one mapping + * which can't be treated in this way for obvious reasons. + */ + if (IS_ENABLED(CONFIG_X86_64) && cpu_feature_enabled(X86_FEATURE_PTI)) + set_memory_np_noalias(begin_ul, len_pages); +} + void __ref free_initmem(void) { e820__reallocate_tables(); - free_init_pages("unused kernel", - (unsigned long)(&__init_begin), - (unsigned long)(&__init_end)); + free_kernel_image_pages(&__init_begin, &__init_end); } #ifdef CONFIG_BLK_DEV_INITRD @@ -880,3 +913,24 @@ void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) __cachemode2pte_tbl[cache] = __cm_idx2pte(entry); __pte2cachemode_tbl[entry] = cache; } + +unsigned long max_swapfile_size(void) +{ + unsigned long pages; + + pages = generic_max_swapfile_size(); + + if (boot_cpu_has_bug(X86_BUG_L1TF)) { + /* Limit the swap file size to MAX_PA/2 for L1TF workaround */ + unsigned long l1tf_limit = l1tf_pfn_limit() + 1; + /* + * We encode swap offsets also with 3 bits below those for pfn + * which makes the usable limit higher. + */ +#if CONFIG_PGTABLE_LEVELS > 2 + l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT; +#endif + pages = min_t(unsigned long, l1tf_limit, pages); + } + return pages; +} diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index a688617c727e..dd519f372169 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -1283,20 +1283,10 @@ void mark_rodata_ro(void) set_memory_ro(start, (end-start) >> PAGE_SHIFT); #endif - free_init_pages("unused kernel", - (unsigned long) __va(__pa_symbol(text_end)), - (unsigned long) __va(__pa_symbol(rodata_start))); - free_init_pages("unused kernel", - (unsigned long) __va(__pa_symbol(rodata_end)), - (unsigned long) __va(__pa_symbol(_sdata))); + free_kernel_image_pages((void *)text_end, (void *)rodata_start); + free_kernel_image_pages((void *)rodata_end, (void *)_sdata); debug_checkwx(); - - /* - * Do this after all of the manipulation of the - * kernel text page tables are complete. - */ - pti_clone_kernel_text(); } int kern_addr_valid(unsigned long addr) diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c index 7c8686709636..79eb55ce69a9 100644 --- a/arch/x86/mm/kmmio.c +++ b/arch/x86/mm/kmmio.c @@ -126,24 +126,29 @@ static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long addr) static void clear_pmd_presence(pmd_t *pmd, bool clear, pmdval_t *old) { + pmd_t new_pmd; pmdval_t v = pmd_val(*pmd); if (clear) { - *old = v & _PAGE_PRESENT; - v &= ~_PAGE_PRESENT; - } else /* presume this has been called with clear==true previously */ - v |= *old; - set_pmd(pmd, __pmd(v)); + *old = v; + new_pmd = pmd_mknotpresent(*pmd); + } else { + /* Presume this has been called with clear==true previously */ + new_pmd = __pmd(*old); + } + set_pmd(pmd, new_pmd); } static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old) { pteval_t v = pte_val(*pte); if (clear) { - *old = v & _PAGE_PRESENT; - v &= ~_PAGE_PRESENT; - } else /* presume this has been called with clear==true previously */ - v |= *old; - set_pte_atomic(pte, __pte(v)); + *old = v; + /* Nothing should care about address */ + pte_clear(&init_mm, 0, pte); + } else { + /* Presume this has been called with clear==true previously */ + set_pte_atomic(pte, __pte(*old)); + } } static int clear_page_presence(struct kmmio_fault_page *f, bool clear) diff --git a/arch/x86/mm/mmap.c b/arch/x86/mm/mmap.c index 48c591251600..f40ab8185d94 100644 --- a/arch/x86/mm/mmap.c +++ b/arch/x86/mm/mmap.c @@ -240,3 +240,24 @@ int valid_mmap_phys_addr_range(unsigned long pfn, size_t count) return phys_addr_valid(addr + count - 1); } + +/* + * Only allow root to set high MMIO mappings to PROT_NONE. + * This prevents an unpriv. user to set them to PROT_NONE and invert + * them, then pointing to valid memory for L1TF speculation. + * + * Note: for locked down kernels may want to disable the root override. + */ +bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) +{ + if (!boot_cpu_has_bug(X86_BUG_L1TF)) + return true; + if (!__pte_needs_invert(pgprot_val(prot))) + return true; + /* If it's real memory always allow */ + if (pfn_valid(pfn)) + return true; + if (pfn > l1tf_pfn_limit() && !capable(CAP_SYS_ADMIN)) + return false; + return true; +} diff --git a/arch/x86/mm/numa_emulation.c b/arch/x86/mm/numa_emulation.c index 34a2a3bfde9c..b54d52a2d00a 100644 --- a/arch/x86/mm/numa_emulation.c +++ b/arch/x86/mm/numa_emulation.c @@ -61,7 +61,7 @@ static int __init emu_setup_memblk(struct numa_meminfo *ei, eb->nid = nid; if (emu_nid_to_phys[nid] == NUMA_NO_NODE) - emu_nid_to_phys[nid] = nid; + emu_nid_to_phys[nid] = pb->nid; pb->start += size; if (pb->start >= pb->end) { @@ -198,40 +198,73 @@ static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size) return end; } +static u64 uniform_size(u64 max_addr, u64 base, u64 hole, int nr_nodes) +{ + unsigned long max_pfn = PHYS_PFN(max_addr); + unsigned long base_pfn = PHYS_PFN(base); + unsigned long hole_pfns = PHYS_PFN(hole); + + return PFN_PHYS((max_pfn - base_pfn - hole_pfns) / nr_nodes); +} + /* * Sets up fake nodes of `size' interleaved over physical nodes ranging from * `addr' to `max_addr'. * * Returns zero on success or negative on error. */ -static int __init split_nodes_size_interleave(struct numa_meminfo *ei, +static int __init split_nodes_size_interleave_uniform(struct numa_meminfo *ei, struct numa_meminfo *pi, - u64 addr, u64 max_addr, u64 size) + u64 addr, u64 max_addr, u64 size, + int nr_nodes, struct numa_memblk *pblk, + int nid) { nodemask_t physnode_mask = numa_nodes_parsed; + int i, ret, uniform = 0; u64 min_size; - int nid = 0; - int i, ret; - if (!size) + if ((!size && !nr_nodes) || (nr_nodes && !pblk)) return -1; + /* - * The limit on emulated nodes is MAX_NUMNODES, so the size per node is - * increased accordingly if the requested size is too small. This - * creates a uniform distribution of node sizes across the entire - * machine (but not necessarily over physical nodes). + * In the 'uniform' case split the passed in physical node by + * nr_nodes, in the non-uniform case, ignore the passed in + * physical block and try to create nodes of at least size + * @size. + * + * In the uniform case, split the nodes strictly by physical + * capacity, i.e. ignore holes. In the non-uniform case account + * for holes and treat @size as a minimum floor. */ - min_size = (max_addr - addr - mem_hole_size(addr, max_addr)) / MAX_NUMNODES; - min_size = max(min_size, FAKE_NODE_MIN_SIZE); - if ((min_size & FAKE_NODE_MIN_HASH_MASK) < min_size) - min_size = (min_size + FAKE_NODE_MIN_SIZE) & - FAKE_NODE_MIN_HASH_MASK; + if (!nr_nodes) + nr_nodes = MAX_NUMNODES; + else { + nodes_clear(physnode_mask); + node_set(pblk->nid, physnode_mask); + uniform = 1; + } + + if (uniform) { + min_size = uniform_size(max_addr, addr, 0, nr_nodes); + size = min_size; + } else { + /* + * The limit on emulated nodes is MAX_NUMNODES, so the + * size per node is increased accordingly if the + * requested size is too small. This creates a uniform + * distribution of node sizes across the entire machine + * (but not necessarily over physical nodes). + */ + min_size = uniform_size(max_addr, addr, + mem_hole_size(addr, max_addr), nr_nodes); + } + min_size = ALIGN(max(min_size, FAKE_NODE_MIN_SIZE), FAKE_NODE_MIN_SIZE); if (size < min_size) { pr_err("Fake node size %LuMB too small, increasing to %LuMB\n", size >> 20, min_size >> 20); size = min_size; } - size &= FAKE_NODE_MIN_HASH_MASK; + size = ALIGN_DOWN(size, FAKE_NODE_MIN_SIZE); /* * Fill physical nodes with fake nodes of size until there is no memory @@ -248,10 +281,14 @@ static int __init split_nodes_size_interleave(struct numa_meminfo *ei, node_clear(i, physnode_mask); continue; } + start = pi->blk[phys_blk].start; limit = pi->blk[phys_blk].end; - end = find_end_of_node(start, limit, size); + if (uniform) + end = start + size; + else + end = find_end_of_node(start, limit, size); /* * If there won't be at least FAKE_NODE_MIN_SIZE of * non-reserved memory in ZONE_DMA32 for the next node, @@ -266,7 +303,8 @@ static int __init split_nodes_size_interleave(struct numa_meminfo *ei, * next node, this one must extend to the end of the * physical node. */ - if (limit - end - mem_hole_size(end, limit) < size) + if ((limit - end - mem_hole_size(end, limit) < size) + && !uniform) end = limit; ret = emu_setup_memblk(ei, pi, nid++ % MAX_NUMNODES, @@ -276,7 +314,15 @@ static int __init split_nodes_size_interleave(struct numa_meminfo *ei, return ret; } } - return 0; + return nid; +} + +static int __init split_nodes_size_interleave(struct numa_meminfo *ei, + struct numa_meminfo *pi, + u64 addr, u64 max_addr, u64 size) +{ + return split_nodes_size_interleave_uniform(ei, pi, addr, max_addr, size, + 0, NULL, NUMA_NO_NODE); } int __init setup_emu2phys_nid(int *dfl_phys_nid) @@ -346,7 +392,28 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) * the fixed node size. Otherwise, if it is just a single number N, * split the system RAM into N fake nodes. */ - if (strchr(emu_cmdline, 'M') || strchr(emu_cmdline, 'G')) { + if (strchr(emu_cmdline, 'U')) { + nodemask_t physnode_mask = numa_nodes_parsed; + unsigned long n; + int nid = 0; + + n = simple_strtoul(emu_cmdline, &emu_cmdline, 0); + ret = -1; + for_each_node_mask(i, physnode_mask) { + ret = split_nodes_size_interleave_uniform(&ei, &pi, + pi.blk[i].start, pi.blk[i].end, 0, + n, &pi.blk[i], nid); + if (ret < 0) + break; + if (ret < n) { + pr_info("%s: phys: %d only got %d of %ld nodes, failing\n", + __func__, i, ret, n); + ret = -1; + break; + } + nid = ret; + } + } else if (strchr(emu_cmdline, 'M') || strchr(emu_cmdline, 'G')) { u64 size; size = memparse(emu_cmdline, &emu_cmdline); diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 3bded76e8d5c..8d6c34fe49be 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -53,6 +53,7 @@ static DEFINE_SPINLOCK(cpa_lock); #define CPA_FLUSHTLB 1 #define CPA_ARRAY 2 #define CPA_PAGES_ARRAY 4 +#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */ #ifdef CONFIG_PROC_FS static unsigned long direct_pages_count[PG_LEVEL_NUM]; @@ -1014,8 +1015,8 @@ static long populate_pmd(struct cpa_data *cpa, pmd = pmd_offset(pud, start); - set_pmd(pmd, __pmd(cpa->pfn << PAGE_SHIFT | _PAGE_PSE | - massage_pgprot(pmd_pgprot))); + set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn, + canon_pgprot(pmd_pgprot)))); start += PMD_SIZE; cpa->pfn += PMD_SIZE >> PAGE_SHIFT; @@ -1087,8 +1088,8 @@ static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d, * Map everything starting from the Gb boundary, possibly with 1G pages */ while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) { - set_pud(pud, __pud(cpa->pfn << PAGE_SHIFT | _PAGE_PSE | - massage_pgprot(pud_pgprot))); + set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn, + canon_pgprot(pud_pgprot)))); start += PUD_SIZE; cpa->pfn += PUD_SIZE >> PAGE_SHIFT; @@ -1486,6 +1487,9 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, /* No alias checking for _NX bit modifications */ checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX; + /* Has caller explicitly disabled alias checking? */ + if (in_flag & CPA_NO_CHECK_ALIAS) + checkalias = 0; ret = __change_page_attr_set_clr(&cpa, checkalias); @@ -1772,6 +1776,15 @@ int set_memory_np(unsigned long addr, int numpages) return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0); } +int set_memory_np_noalias(unsigned long addr, int numpages) +{ + int cpa_flags = CPA_NO_CHECK_ALIAS; + + return change_page_attr_set_clr(&addr, numpages, __pgprot(0), + __pgprot(_PAGE_PRESENT), 0, + cpa_flags, NULL); +} + int set_memory_4k(unsigned long addr, int numpages) { return change_page_attr_set_clr(&addr, numpages, __pgprot(0), @@ -1784,6 +1797,12 @@ int set_memory_nonglobal(unsigned long addr, int numpages) __pgprot(_PAGE_GLOBAL), 0); } +int set_memory_global(unsigned long addr, int numpages) +{ + return change_page_attr_set(&addr, numpages, + __pgprot(_PAGE_GLOBAL), 0); +} + static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) { struct cpa_data cpa; diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 47b5951e592b..3ef095c70ae3 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -182,6 +182,14 @@ static void pgd_dtor(pgd_t *pgd) */ #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD +/* + * We allocate separate PMDs for the kernel part of the user page-table + * when PTI is enabled. We need them to map the per-process LDT into the + * user-space page-table. + */ +#define PREALLOCATED_USER_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \ + KERNEL_PGD_PTRS : 0) + void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) { paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); @@ -202,14 +210,14 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) /* No need to prepopulate any pagetable entries in non-PAE modes. */ #define PREALLOCATED_PMDS 0 - +#define PREALLOCATED_USER_PMDS 0 #endif /* CONFIG_X86_PAE */ -static void free_pmds(struct mm_struct *mm, pmd_t *pmds[]) +static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) { int i; - for(i = 0; i < PREALLOCATED_PMDS; i++) + for (i = 0; i < count; i++) if (pmds[i]) { pgtable_pmd_page_dtor(virt_to_page(pmds[i])); free_page((unsigned long)pmds[i]); @@ -217,7 +225,7 @@ static void free_pmds(struct mm_struct *mm, pmd_t *pmds[]) } } -static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) +static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) { int i; bool failed = false; @@ -226,7 +234,7 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) if (mm == &init_mm) gfp &= ~__GFP_ACCOUNT; - for(i = 0; i < PREALLOCATED_PMDS; i++) { + for (i = 0; i < count; i++) { pmd_t *pmd = (pmd_t *)__get_free_page(gfp); if (!pmd) failed = true; @@ -241,7 +249,7 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) } if (failed) { - free_pmds(mm, pmds); + free_pmds(mm, pmds, count); return -ENOMEM; } @@ -254,23 +262,38 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[]) * preallocate which never got a corresponding vma will need to be * freed manually. */ +static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) +{ + pgd_t pgd = *pgdp; + + if (pgd_val(pgd) != 0) { + pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); + + *pgdp = native_make_pgd(0); + + paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); + pmd_free(mm, pmd); + mm_dec_nr_pmds(mm); + } +} + static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) { int i; - for(i = 0; i < PREALLOCATED_PMDS; i++) { - pgd_t pgd = pgdp[i]; + for (i = 0; i < PREALLOCATED_PMDS; i++) + mop_up_one_pmd(mm, &pgdp[i]); - if (pgd_val(pgd) != 0) { - pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); +#ifdef CONFIG_PAGE_TABLE_ISOLATION - pgdp[i] = native_make_pgd(0); + if (!static_cpu_has(X86_FEATURE_PTI)) + return; - paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); - pmd_free(mm, pmd); - mm_dec_nr_pmds(mm); - } - } + pgdp = kernel_to_user_pgdp(pgdp); + + for (i = 0; i < PREALLOCATED_USER_PMDS; i++) + mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); +#endif } static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[]) @@ -296,6 +319,38 @@ static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[]) } } +#ifdef CONFIG_PAGE_TABLE_ISOLATION +static void pgd_prepopulate_user_pmd(struct mm_struct *mm, + pgd_t *k_pgd, pmd_t *pmds[]) +{ + pgd_t *s_pgd = kernel_to_user_pgdp(swapper_pg_dir); + pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); + p4d_t *u_p4d; + pud_t *u_pud; + int i; + + u_p4d = p4d_offset(u_pgd, 0); + u_pud = pud_offset(u_p4d, 0); + + s_pgd += KERNEL_PGD_BOUNDARY; + u_pud += KERNEL_PGD_BOUNDARY; + + for (i = 0; i < PREALLOCATED_USER_PMDS; i++, u_pud++, s_pgd++) { + pmd_t *pmd = pmds[i]; + + memcpy(pmd, (pmd_t *)pgd_page_vaddr(*s_pgd), + sizeof(pmd_t) * PTRS_PER_PMD); + + pud_populate(mm, u_pud, pmd); + } + +} +#else +static void pgd_prepopulate_user_pmd(struct mm_struct *mm, + pgd_t *k_pgd, pmd_t *pmds[]) +{ +} +#endif /* * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also * assumes that pgd should be in one page. @@ -329,9 +384,6 @@ static int __init pgd_cache_init(void) */ pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN, SLAB_PANIC, NULL); - if (!pgd_cache) - return -ENOMEM; - return 0; } core_initcall(pgd_cache_init); @@ -343,7 +395,8 @@ static inline pgd_t *_pgd_alloc(void) * We allocate one page for pgd. */ if (!SHARED_KERNEL_PMD) - return (pgd_t *)__get_free_page(PGALLOC_GFP); + return (pgd_t *)__get_free_pages(PGALLOC_GFP, + PGD_ALLOCATION_ORDER); /* * Now PAE kernel is not running as a Xen domain. We can allocate @@ -355,7 +408,7 @@ static inline pgd_t *_pgd_alloc(void) static inline void _pgd_free(pgd_t *pgd) { if (!SHARED_KERNEL_PMD) - free_page((unsigned long)pgd); + free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); else kmem_cache_free(pgd_cache, pgd); } @@ -375,6 +428,7 @@ static inline void _pgd_free(pgd_t *pgd) pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *pgd; + pmd_t *u_pmds[PREALLOCATED_USER_PMDS]; pmd_t *pmds[PREALLOCATED_PMDS]; pgd = _pgd_alloc(); @@ -384,12 +438,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm) mm->pgd = pgd; - if (preallocate_pmds(mm, pmds) != 0) + if (preallocate_pmds(mm, pmds, PREALLOCATED_PMDS) != 0) goto out_free_pgd; - if (paravirt_pgd_alloc(mm) != 0) + if (preallocate_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS) != 0) goto out_free_pmds; + if (paravirt_pgd_alloc(mm) != 0) + goto out_free_user_pmds; + /* * Make sure that pre-populating the pmds is atomic with * respect to anything walking the pgd_list, so that they @@ -399,13 +456,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm) pgd_ctor(mm, pgd); pgd_prepopulate_pmd(mm, pgd, pmds); + pgd_prepopulate_user_pmd(mm, pgd, u_pmds); spin_unlock(&pgd_lock); return pgd; +out_free_user_pmds: + free_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS); out_free_pmds: - free_pmds(mm, pmds); + free_pmds(mm, pmds, PREALLOCATED_PMDS); out_free_pgd: _pgd_free(pgd); out: @@ -719,28 +779,50 @@ int pmd_clear_huge(pmd_t *pmd) return 0; } +#ifdef CONFIG_X86_64 /** * pud_free_pmd_page - Clear pud entry and free pmd page. * @pud: Pointer to a PUD. + * @addr: Virtual address associated with pud. * - * Context: The pud range has been unmaped and TLB purged. + * Context: The pud range has been unmapped and TLB purged. * Return: 1 if clearing the entry succeeded. 0 otherwise. + * + * NOTE: Callers must allow a single page allocation. */ -int pud_free_pmd_page(pud_t *pud) +int pud_free_pmd_page(pud_t *pud, unsigned long addr) { - pmd_t *pmd; + pmd_t *pmd, *pmd_sv; + pte_t *pte; int i; if (pud_none(*pud)) return 1; pmd = (pmd_t *)pud_page_vaddr(*pud); + pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL); + if (!pmd_sv) + return 0; - for (i = 0; i < PTRS_PER_PMD; i++) - if (!pmd_free_pte_page(&pmd[i])) - return 0; + for (i = 0; i < PTRS_PER_PMD; i++) { + pmd_sv[i] = pmd[i]; + if (!pmd_none(pmd[i])) + pmd_clear(&pmd[i]); + } pud_clear(pud); + + /* INVLPG to clear all paging-structure caches */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); + + for (i = 0; i < PTRS_PER_PMD; i++) { + if (!pmd_none(pmd_sv[i])) { + pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); + free_page((unsigned long)pte); + } + } + + free_page((unsigned long)pmd_sv); free_page((unsigned long)pmd); return 1; @@ -749,11 +831,12 @@ int pud_free_pmd_page(pud_t *pud) /** * pmd_free_pte_page - Clear pmd entry and free pte page. * @pmd: Pointer to a PMD. + * @addr: Virtual address associated with pmd. * - * Context: The pmd range has been unmaped and TLB purged. + * Context: The pmd range has been unmapped and TLB purged. * Return: 1 if clearing the entry succeeded. 0 otherwise. */ -int pmd_free_pte_page(pmd_t *pmd) +int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) { pte_t *pte; @@ -762,8 +845,30 @@ int pmd_free_pte_page(pmd_t *pmd) pte = (pte_t *)pmd_page_vaddr(*pmd); pmd_clear(pmd); + + /* INVLPG to clear all paging-structure caches */ + flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); + free_page((unsigned long)pte); return 1; } + +#else /* !CONFIG_X86_64 */ + +int pud_free_pmd_page(pud_t *pud, unsigned long addr) +{ + return pud_none(*pud); +} + +/* + * Disable free page handling on x86-PAE. This assures that ioremap() + * does not update sync'd pmd entries. See vmalloc_sync_one(). + */ +int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) +{ + return pmd_none(*pmd); +} + +#endif /* CONFIG_X86_64 */ #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index 4d418e705878..31341ae7309f 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -45,6 +45,7 @@ #include <asm/pgalloc.h> #include <asm/tlbflush.h> #include <asm/desc.h> +#include <asm/sections.h> #undef pr_fmt #define pr_fmt(fmt) "Kernel/User page tables isolation: " fmt @@ -54,6 +55,16 @@ #define __GFP_NOTRACK 0 #endif +/* + * Define the page-table levels we clone for user-space on 32 + * and 64 bit. + */ +#ifdef CONFIG_X86_64 +#define PTI_LEVEL_KERNEL_IMAGE PTI_CLONE_PMD +#else +#define PTI_LEVEL_KERNEL_IMAGE PTI_CLONE_PTE +#endif + static void __init pti_print_if_insecure(const char *reason) { if (boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) @@ -117,7 +128,7 @@ enable: setup_force_cpu_cap(X86_FEATURE_PTI); } -pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd) +pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) { /* * Changes to the high (kernel) portion of the kernelmode page @@ -176,7 +187,7 @@ static p4d_t *pti_user_pagetable_walk_p4d(unsigned long address) if (pgd_none(*pgd)) { unsigned long new_p4d_page = __get_free_page(gfp); - if (!new_p4d_page) + if (WARN_ON_ONCE(!new_p4d_page)) return NULL; set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); @@ -195,13 +206,17 @@ static p4d_t *pti_user_pagetable_walk_p4d(unsigned long address) static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) { gfp_t gfp = (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); - p4d_t *p4d = pti_user_pagetable_walk_p4d(address); + p4d_t *p4d; pud_t *pud; + p4d = pti_user_pagetable_walk_p4d(address); + if (!p4d) + return NULL; + BUILD_BUG_ON(p4d_large(*p4d) != 0); if (p4d_none(*p4d)) { unsigned long new_pud_page = __get_free_page(gfp); - if (!new_pud_page) + if (WARN_ON_ONCE(!new_pud_page)) return NULL; set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); @@ -215,7 +230,7 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) } if (pud_none(*pud)) { unsigned long new_pmd_page = __get_free_page(gfp); - if (!new_pmd_page) + if (WARN_ON_ONCE(!new_pmd_page)) return NULL; set_pud(pud, __pud(_KERNPG_TABLE | __pa(new_pmd_page))); @@ -224,7 +239,6 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) return pmd_offset(pud, address); } -#ifdef CONFIG_X86_VSYSCALL_EMULATION /* * Walk the shadow copy of the page tables (optionally) trying to allocate * page table pages on the way down. Does not support large pages. @@ -237,9 +251,13 @@ static pmd_t *pti_user_pagetable_walk_pmd(unsigned long address) static __init pte_t *pti_user_pagetable_walk_pte(unsigned long address) { gfp_t gfp = (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); - pmd_t *pmd = pti_user_pagetable_walk_pmd(address); + pmd_t *pmd; pte_t *pte; + pmd = pti_user_pagetable_walk_pmd(address); + if (!pmd) + return NULL; + /* We can't do anything sensible if we hit a large mapping. */ if (pmd_large(*pmd)) { WARN_ON(1); @@ -262,6 +280,7 @@ static __init pte_t *pti_user_pagetable_walk_pte(unsigned long address) return pte; } +#ifdef CONFIG_X86_VSYSCALL_EMULATION static void __init pti_setup_vsyscall(void) { pte_t *pte, *target_pte; @@ -282,8 +301,14 @@ static void __init pti_setup_vsyscall(void) static void __init pti_setup_vsyscall(void) { } #endif +enum pti_clone_level { + PTI_CLONE_PMD, + PTI_CLONE_PTE, +}; + static void -pti_clone_pmds(unsigned long start, unsigned long end, pmdval_t clear) +pti_clone_pgtable(unsigned long start, unsigned long end, + enum pti_clone_level level) { unsigned long addr; @@ -291,59 +316,105 @@ pti_clone_pmds(unsigned long start, unsigned long end, pmdval_t clear) * Clone the populated PMDs which cover start to end. These PMD areas * can have holes. */ - for (addr = start; addr < end; addr += PMD_SIZE) { + for (addr = start; addr < end;) { + pte_t *pte, *target_pte; pmd_t *pmd, *target_pmd; pgd_t *pgd; p4d_t *p4d; pud_t *pud; + /* Overflow check */ + if (addr < start) + break; + pgd = pgd_offset_k(addr); if (WARN_ON(pgd_none(*pgd))) return; p4d = p4d_offset(pgd, addr); if (WARN_ON(p4d_none(*p4d))) return; + pud = pud_offset(p4d, addr); - if (pud_none(*pud)) + if (pud_none(*pud)) { + addr += PUD_SIZE; continue; + } + pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) + if (pmd_none(*pmd)) { + addr += PMD_SIZE; continue; + } - target_pmd = pti_user_pagetable_walk_pmd(addr); - if (WARN_ON(!target_pmd)) - return; - - /* - * Only clone present PMDs. This ensures only setting - * _PAGE_GLOBAL on present PMDs. This should only be - * called on well-known addresses anyway, so a non- - * present PMD would be a surprise. - */ - if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT))) - return; - - /* - * Setting 'target_pmd' below creates a mapping in both - * the user and kernel page tables. It is effectively - * global, so set it as global in both copies. Note: - * the X86_FEATURE_PGE check is not _required_ because - * the CPU ignores _PAGE_GLOBAL when PGE is not - * supported. The check keeps consistentency with - * code that only set this bit when supported. - */ - if (boot_cpu_has(X86_FEATURE_PGE)) - *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); - - /* - * Copy the PMD. That is, the kernelmode and usermode - * tables will share the last-level page tables of this - * address range - */ - *target_pmd = pmd_clear_flags(*pmd, clear); + if (pmd_large(*pmd) || level == PTI_CLONE_PMD) { + target_pmd = pti_user_pagetable_walk_pmd(addr); + if (WARN_ON(!target_pmd)) + return; + + /* + * Only clone present PMDs. This ensures only setting + * _PAGE_GLOBAL on present PMDs. This should only be + * called on well-known addresses anyway, so a non- + * present PMD would be a surprise. + */ + if (WARN_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT))) + return; + + /* + * Setting 'target_pmd' below creates a mapping in both + * the user and kernel page tables. It is effectively + * global, so set it as global in both copies. Note: + * the X86_FEATURE_PGE check is not _required_ because + * the CPU ignores _PAGE_GLOBAL when PGE is not + * supported. The check keeps consistentency with + * code that only set this bit when supported. + */ + if (boot_cpu_has(X86_FEATURE_PGE)) + *pmd = pmd_set_flags(*pmd, _PAGE_GLOBAL); + + /* + * Copy the PMD. That is, the kernelmode and usermode + * tables will share the last-level page tables of this + * address range + */ + *target_pmd = *pmd; + + addr += PMD_SIZE; + + } else if (level == PTI_CLONE_PTE) { + + /* Walk the page-table down to the pte level */ + pte = pte_offset_kernel(pmd, addr); + if (pte_none(*pte)) { + addr += PAGE_SIZE; + continue; + } + + /* Only clone present PTEs */ + if (WARN_ON(!(pte_flags(*pte) & _PAGE_PRESENT))) + return; + + /* Allocate PTE in the user page-table */ + target_pte = pti_user_pagetable_walk_pte(addr); + if (WARN_ON(!target_pte)) + return; + + /* Set GLOBAL bit in both PTEs */ + if (boot_cpu_has(X86_FEATURE_PGE)) + *pte = pte_set_flags(*pte, _PAGE_GLOBAL); + + /* Clone the PTE */ + *target_pte = *pte; + + addr += PAGE_SIZE; + + } else { + BUG(); + } } } +#ifdef CONFIG_X86_64 /* * Clone a single p4d (i.e. a top-level entry on 4-level systems and a * next-level entry on 5-level systems. @@ -354,6 +425,9 @@ static void __init pti_clone_p4d(unsigned long addr) pgd_t *kernel_pgd; user_p4d = pti_user_pagetable_walk_p4d(addr); + if (!user_p4d) + return; + kernel_pgd = pgd_offset_k(addr); kernel_p4d = p4d_offset(kernel_pgd, addr); *user_p4d = *kernel_p4d; @@ -367,6 +441,25 @@ static void __init pti_clone_user_shared(void) pti_clone_p4d(CPU_ENTRY_AREA_BASE); } +#else /* CONFIG_X86_64 */ + +/* + * On 32 bit PAE systems with 1GB of Kernel address space there is only + * one pgd/p4d for the whole kernel. Cloning that would map the whole + * address space into the user page-tables, making PTI useless. So clone + * the page-table on the PMD level to prevent that. + */ +static void __init pti_clone_user_shared(void) +{ + unsigned long start, end; + + start = CPU_ENTRY_AREA_BASE; + end = start + (PAGE_SIZE * CPU_ENTRY_AREA_PAGES); + + pti_clone_pgtable(start, end, PTI_CLONE_PMD); +} +#endif /* CONFIG_X86_64 */ + /* * Clone the ESPFIX P4D into the user space visible page table */ @@ -380,11 +473,11 @@ static void __init pti_setup_espfix64(void) /* * Clone the populated PMDs of the entry and irqentry text and force it RO. */ -static void __init pti_clone_entry_text(void) +static void pti_clone_entry_text(void) { - pti_clone_pmds((unsigned long) __entry_text_start, - (unsigned long) __irqentry_text_end, - _PAGE_RW); + pti_clone_pgtable((unsigned long) __entry_text_start, + (unsigned long) __irqentry_text_end, + PTI_CLONE_PMD); } /* @@ -435,10 +528,17 @@ static inline bool pti_kernel_image_global_ok(void) } /* + * This is the only user for these and it is not arch-generic + * like the other set_memory.h functions. Just extern them. + */ +extern int set_memory_nonglobal(unsigned long addr, int numpages); +extern int set_memory_global(unsigned long addr, int numpages); + +/* * For some configurations, map all of kernel text into the user page * tables. This reduces TLB misses, especially on non-PCID systems. */ -void pti_clone_kernel_text(void) +static void pti_clone_kernel_text(void) { /* * rodata is part of the kernel image and is normally @@ -446,7 +546,8 @@ void pti_clone_kernel_text(void) * clone the areas past rodata, they might contain secrets. */ unsigned long start = PFN_ALIGN(_text); - unsigned long end = (unsigned long)__end_rodata_hpage_align; + unsigned long end_clone = (unsigned long)__end_rodata_aligned; + unsigned long end_global = PFN_ALIGN((unsigned long)__stop___ex_table); if (!pti_kernel_image_global_ok()) return; @@ -458,14 +559,18 @@ void pti_clone_kernel_text(void) * pti_set_kernel_image_nonglobal() did to clear the * global bit. */ - pti_clone_pmds(start, end, _PAGE_RW); + pti_clone_pgtable(start, end_clone, PTI_LEVEL_KERNEL_IMAGE); + + /* + * pti_clone_pgtable() will set the global bit in any PMDs + * that it clones, but we also need to get any PTEs in + * the last level for areas that are not huge-page-aligned. + */ + + /* Set the global bit for normal non-__init kernel text: */ + set_memory_global(start, (end_global - start) >> PAGE_SHIFT); } -/* - * This is the only user for it and it is not arch-generic like - * the other set_memory.h functions. Just extern it. - */ -extern int set_memory_nonglobal(unsigned long addr, int numpages); void pti_set_kernel_image_nonglobal(void) { /* @@ -477,9 +582,11 @@ void pti_set_kernel_image_nonglobal(void) unsigned long start = PFN_ALIGN(_text); unsigned long end = ALIGN((unsigned long)_end, PMD_PAGE_SIZE); - if (pti_kernel_image_global_ok()) - return; - + /* + * This clears _PAGE_GLOBAL from the entire kernel image. + * pti_clone_kernel_text() map put _PAGE_GLOBAL back for + * areas that are mapped to userspace. + */ set_memory_nonglobal(start, (end - start) >> PAGE_SHIFT); } @@ -493,6 +600,28 @@ void __init pti_init(void) pr_info("enabled\n"); +#ifdef CONFIG_X86_32 + /* + * We check for X86_FEATURE_PCID here. But the init-code will + * clear the feature flag on 32 bit because the feature is not + * supported on 32 bit anyway. To print the warning we need to + * check with cpuid directly again. + */ + if (cpuid_ecx(0x1) & BIT(17)) { + /* Use printk to work around pr_fmt() */ + printk(KERN_WARNING "\n"); + printk(KERN_WARNING "************************************************************\n"); + printk(KERN_WARNING "** WARNING! WARNING! WARNING! WARNING! WARNING! WARNING! **\n"); + printk(KERN_WARNING "** **\n"); + printk(KERN_WARNING "** You are using 32-bit PTI on a 64-bit PCID-capable CPU. **\n"); + printk(KERN_WARNING "** Your performance will increase dramatically if you **\n"); + printk(KERN_WARNING "** switch to a 64-bit kernel! **\n"); + printk(KERN_WARNING "** **\n"); + printk(KERN_WARNING "** WARNING! WARNING! WARNING! WARNING! WARNING! WARNING! **\n"); + printk(KERN_WARNING "************************************************************\n"); + } +#endif + pti_clone_user_shared(); /* Undo all global bits from the init pagetables in head_64.S: */ @@ -502,3 +631,22 @@ void __init pti_init(void) pti_setup_espfix64(); pti_setup_vsyscall(); } + +/* + * Finalize the kernel mappings in the userspace page-table. Some of the + * mappings for the kernel image might have changed since pti_init() + * cloned them. This is because parts of the kernel image have been + * mapped RO and/or NX. These changes need to be cloned again to the + * userspace page-table. + */ +void pti_finalize(void) +{ + /* + * We need to clone everything (again) that maps parts of the + * kernel image. + */ + pti_clone_entry_text(); + pti_clone_kernel_text(); + + debug_checkwx_user(); +} diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 6eb1f34c3c85..752dbf4e0e50 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -7,6 +7,7 @@ #include <linux/export.h> #include <linux/cpu.h> #include <linux/debugfs.h> +#include <linux/gfp.h> #include <asm/tlbflush.h> #include <asm/mmu_context.h> @@ -35,7 +36,7 @@ * necessary invalidation by clearing out the 'ctx_id' which * forces a TLB flush when the context is loaded. */ -void clear_asid_other(void) +static void clear_asid_other(void) { u16 asid; @@ -185,8 +186,11 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, { struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm); u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); + bool was_lazy = this_cpu_read(cpu_tlbstate.is_lazy); unsigned cpu = smp_processor_id(); u64 next_tlb_gen; + bool need_flush; + u16 new_asid; /* * NB: The scheduler will call us with prev == next when switching @@ -240,20 +244,41 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, next->context.ctx_id); /* - * We don't currently support having a real mm loaded without - * our cpu set in mm_cpumask(). We have all the bookkeeping - * in place to figure out whether we would need to flush - * if our cpu were cleared in mm_cpumask(), but we don't - * currently use it. + * Even in lazy TLB mode, the CPU should stay set in the + * mm_cpumask. The TLB shootdown code can figure out from + * from cpu_tlbstate.is_lazy whether or not to send an IPI. */ if (WARN_ON_ONCE(real_prev != &init_mm && !cpumask_test_cpu(cpu, mm_cpumask(next)))) cpumask_set_cpu(cpu, mm_cpumask(next)); - return; + /* + * If the CPU is not in lazy TLB mode, we are just switching + * from one thread in a process to another thread in the same + * process. No TLB flush required. + */ + if (!was_lazy) + return; + + /* + * Read the tlb_gen to check whether a flush is needed. + * If the TLB is up to date, just use it. + * The barrier synchronizes with the tlb_gen increment in + * the TLB shootdown code. + */ + smp_mb(); + next_tlb_gen = atomic64_read(&next->context.tlb_gen); + if (this_cpu_read(cpu_tlbstate.ctxs[prev_asid].tlb_gen) == + next_tlb_gen) + return; + + /* + * TLB contents went out of date while we were in lazy + * mode. Fall through to the TLB switching code below. + */ + new_asid = prev_asid; + need_flush = true; } else { - u16 new_asid; - bool need_flush; u64 last_ctx_id = this_cpu_read(cpu_tlbstate.last_ctx_id); /* @@ -285,53 +310,60 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, sync_current_stack_to_mm(next); } - /* Stop remote flushes for the previous mm */ - VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu, mm_cpumask(real_prev)) && - real_prev != &init_mm); - cpumask_clear_cpu(cpu, mm_cpumask(real_prev)); + /* + * Stop remote flushes for the previous mm. + * Skip kernel threads; we never send init_mm TLB flushing IPIs, + * but the bitmap manipulation can cause cache line contention. + */ + if (real_prev != &init_mm) { + VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu, + mm_cpumask(real_prev))); + cpumask_clear_cpu(cpu, mm_cpumask(real_prev)); + } /* * Start remote flushes and then read tlb_gen. */ - cpumask_set_cpu(cpu, mm_cpumask(next)); + if (next != &init_mm) + cpumask_set_cpu(cpu, mm_cpumask(next)); next_tlb_gen = atomic64_read(&next->context.tlb_gen); choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); + } - if (need_flush) { - this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); - this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); - load_new_mm_cr3(next->pgd, new_asid, true); - - /* - * NB: This gets called via leave_mm() in the idle path - * where RCU functions differently. Tracing normally - * uses RCU, so we need to use the _rcuidle variant. - * - * (There is no good reason for this. The idle code should - * be rearranged to call this before rcu_idle_enter().) - */ - trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); - } else { - /* The new ASID is already up to date. */ - load_new_mm_cr3(next->pgd, new_asid, false); - - /* See above wrt _rcuidle. */ - trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0); - } + if (need_flush) { + this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); + this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); + load_new_mm_cr3(next->pgd, new_asid, true); /* - * Record last user mm's context id, so we can avoid - * flushing branch buffer with IBPB if we switch back - * to the same user. + * NB: This gets called via leave_mm() in the idle path + * where RCU functions differently. Tracing normally + * uses RCU, so we need to use the _rcuidle variant. + * + * (There is no good reason for this. The idle code should + * be rearranged to call this before rcu_idle_enter().) */ - if (next != &init_mm) - this_cpu_write(cpu_tlbstate.last_ctx_id, next->context.ctx_id); + trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); + } else { + /* The new ASID is already up to date. */ + load_new_mm_cr3(next->pgd, new_asid, false); - this_cpu_write(cpu_tlbstate.loaded_mm, next); - this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid); + /* See above wrt _rcuidle. */ + trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0); } + /* + * Record last user mm's context id, so we can avoid + * flushing branch buffer with IBPB if we switch back + * to the same user. + */ + if (next != &init_mm) + this_cpu_write(cpu_tlbstate.last_ctx_id, next->context.ctx_id); + + this_cpu_write(cpu_tlbstate.loaded_mm, next); + this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid); + load_mm_cr4(next); switch_ldt(real_prev, next); } @@ -354,20 +386,7 @@ void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) return; - if (tlb_defer_switch_to_init_mm()) { - /* - * There's a significant optimization that may be possible - * here. We have accurate enough TLB flush tracking that we - * don't need to maintain coherence of TLB per se when we're - * lazy. We do, however, need to maintain coherence of - * paging-structure caches. We could, in principle, leave our - * old mm loaded and only switch to init_mm when - * tlb_remove_page() happens. - */ - this_cpu_write(cpu_tlbstate.is_lazy, true); - } else { - switch_mm(NULL, &init_mm, NULL); - } + this_cpu_write(cpu_tlbstate.is_lazy, true); } /* @@ -454,6 +473,9 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f, * paging-structure cache to avoid speculatively reading * garbage into our TLB. Since switching to init_mm is barely * slower than a minimal flush, just switch to init_mm. + * + * This should be rare, with native_flush_tlb_others skipping + * IPIs to lazy TLB mode CPUs. */ switch_mm_irqs_off(NULL, &init_mm, NULL); return; @@ -560,6 +582,9 @@ static void flush_tlb_func_remote(void *info) void native_flush_tlb_others(const struct cpumask *cpumask, const struct flush_tlb_info *info) { + cpumask_var_t lazymask; + unsigned int cpu; + count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); if (info->end == TLB_FLUSH_ALL) trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL); @@ -583,8 +608,6 @@ void native_flush_tlb_others(const struct cpumask *cpumask, * that UV should be updated so that smp_call_function_many(), * etc, are optimal on UV. */ - unsigned int cpu; - cpu = smp_processor_id(); cpumask = uv_flush_tlb_others(cpumask, info); if (cpumask) @@ -592,8 +615,29 @@ void native_flush_tlb_others(const struct cpumask *cpumask, (void *)info, 1); return; } - smp_call_function_many(cpumask, flush_tlb_func_remote, + + /* + * A temporary cpumask is used in order to skip sending IPIs + * to CPUs in lazy TLB state, while keeping them in mm_cpumask(mm). + * If the allocation fails, simply IPI every CPU in mm_cpumask. + */ + if (!alloc_cpumask_var(&lazymask, GFP_ATOMIC)) { + smp_call_function_many(cpumask, flush_tlb_func_remote, (void *)info, 1); + return; + } + + cpumask_copy(lazymask, cpumask); + + for_each_cpu(cpu, lazymask) { + if (per_cpu(cpu_tlbstate.is_lazy, cpu)) + cpumask_clear_cpu(cpu, lazymask); + } + + smp_call_function_many(lazymask, flush_tlb_func_remote, + (void *)info, 1); + + free_cpumask_var(lazymask); } /* @@ -646,6 +690,68 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, put_cpu(); } +void tlb_flush_remove_tables_local(void *arg) +{ + struct mm_struct *mm = arg; + + if (this_cpu_read(cpu_tlbstate.loaded_mm) == mm && + this_cpu_read(cpu_tlbstate.is_lazy)) { + /* + * We're in lazy mode. We need to at least flush our + * paging-structure cache to avoid speculatively reading + * garbage into our TLB. Since switching to init_mm is barely + * slower than a minimal flush, just switch to init_mm. + */ + switch_mm_irqs_off(NULL, &init_mm, NULL); + } +} + +static void mm_fill_lazy_tlb_cpu_mask(struct mm_struct *mm, + struct cpumask *lazy_cpus) +{ + int cpu; + + for_each_cpu(cpu, mm_cpumask(mm)) { + if (!per_cpu(cpu_tlbstate.is_lazy, cpu)) + cpumask_set_cpu(cpu, lazy_cpus); + } +} + +void tlb_flush_remove_tables(struct mm_struct *mm) +{ + int cpu = get_cpu(); + cpumask_var_t lazy_cpus; + + if (cpumask_any_but(mm_cpumask(mm), cpu) >= nr_cpu_ids) { + put_cpu(); + return; + } + + if (!zalloc_cpumask_var(&lazy_cpus, GFP_ATOMIC)) { + /* + * If the cpumask allocation fails, do a brute force flush + * on all the CPUs that have this mm loaded. + */ + smp_call_function_many(mm_cpumask(mm), + tlb_flush_remove_tables_local, (void *)mm, 1); + put_cpu(); + return; + } + + /* + * CPUs with !is_lazy either received a TLB flush IPI while the user + * pages in this address range were unmapped, or have context switched + * and reloaded %CR3 since then. + * + * Shootdown IPIs at page table freeing time only need to be sent to + * CPUs that may have out of date TLB contents. + */ + mm_fill_lazy_tlb_cpu_mask(mm, lazy_cpus); + smp_call_function_many(lazy_cpus, + tlb_flush_remove_tables_local, (void *)mm, 1); + free_cpumask_var(lazy_cpus); + put_cpu(); +} static void do_flush_tlb_all(void *info) { diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 55799873ebe5..8f6cc71e0848 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -1441,8 +1441,8 @@ static void emit_prologue(u8 **pprog, u32 stack_depth) /* sub esp,STACK_SIZE */ EMIT2_off32(0x81, 0xEC, STACK_SIZE); - /* sub ebp,SCRATCH_SIZE+4+12*/ - EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 16); + /* sub ebp,SCRATCH_SIZE+12*/ + EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12); /* xor ebx,ebx */ EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX)); @@ -1475,8 +1475,8 @@ static void emit_epilogue(u8 **pprog, u32 stack_depth) /* mov edx,dword ptr [ebp+off]*/ EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1])); - /* add ebp,SCRATCH_SIZE+4+12*/ - EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 16); + /* add ebp,SCRATCH_SIZE+12*/ + EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12); /* mov ebx,dword ptr [ebp-12]*/ EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12); diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 77873ce700ae..ee5d08f25ce4 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -417,7 +417,7 @@ static void __init __map_region(efi_memory_desc_t *md, u64 va) if (!(md->attribute & EFI_MEMORY_WB)) flags |= _PAGE_PCD; - if (sev_active()) + if (sev_active() && md->type != EFI_MEMORY_MAPPED_IO) flags |= _PAGE_ENC; pfn = md->phys_addr >> PAGE_SHIFT; @@ -636,6 +636,8 @@ void efi_switch_mm(struct mm_struct *mm) #ifdef CONFIG_EFI_MIXED extern efi_status_t efi64_thunk(u32, ...); +static DEFINE_SPINLOCK(efi_runtime_lock); + #define runtime_service32(func) \ ({ \ u32 table = (u32)(unsigned long)efi.systab; \ @@ -657,17 +659,14 @@ extern efi_status_t efi64_thunk(u32, ...); #define efi_thunk(f, ...) \ ({ \ efi_status_t __s; \ - unsigned long __flags; \ u32 __func; \ \ - local_irq_save(__flags); \ arch_efi_call_virt_setup(); \ \ __func = runtime_service32(f); \ __s = efi64_thunk(__func, __VA_ARGS__); \ \ arch_efi_call_virt_teardown(); \ - local_irq_restore(__flags); \ \ __s; \ }) @@ -702,14 +701,17 @@ static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc) { efi_status_t status; u32 phys_tm, phys_tc; + unsigned long flags; spin_lock(&rtc_lock); + spin_lock_irqsave(&efi_runtime_lock, flags); phys_tm = virt_to_phys_or_null(tm); phys_tc = virt_to_phys_or_null(tc); status = efi_thunk(get_time, phys_tm, phys_tc); + spin_unlock_irqrestore(&efi_runtime_lock, flags); spin_unlock(&rtc_lock); return status; @@ -719,13 +721,16 @@ static efi_status_t efi_thunk_set_time(efi_time_t *tm) { efi_status_t status; u32 phys_tm; + unsigned long flags; spin_lock(&rtc_lock); + spin_lock_irqsave(&efi_runtime_lock, flags); phys_tm = virt_to_phys_or_null(tm); status = efi_thunk(set_time, phys_tm); + spin_unlock_irqrestore(&efi_runtime_lock, flags); spin_unlock(&rtc_lock); return status; @@ -737,8 +742,10 @@ efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending, { efi_status_t status; u32 phys_enabled, phys_pending, phys_tm; + unsigned long flags; spin_lock(&rtc_lock); + spin_lock_irqsave(&efi_runtime_lock, flags); phys_enabled = virt_to_phys_or_null(enabled); phys_pending = virt_to_phys_or_null(pending); @@ -747,6 +754,7 @@ efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending, status = efi_thunk(get_wakeup_time, phys_enabled, phys_pending, phys_tm); + spin_unlock_irqrestore(&efi_runtime_lock, flags); spin_unlock(&rtc_lock); return status; @@ -757,13 +765,16 @@ efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm) { efi_status_t status; u32 phys_tm; + unsigned long flags; spin_lock(&rtc_lock); + spin_lock_irqsave(&efi_runtime_lock, flags); phys_tm = virt_to_phys_or_null(tm); status = efi_thunk(set_wakeup_time, enabled, phys_tm); + spin_unlock_irqrestore(&efi_runtime_lock, flags); spin_unlock(&rtc_lock); return status; @@ -781,6 +792,9 @@ efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor, efi_status_t status; u32 phys_name, phys_vendor, phys_attr; u32 phys_data_size, phys_data; + unsigned long flags; + + spin_lock_irqsave(&efi_runtime_lock, flags); phys_data_size = virt_to_phys_or_null(data_size); phys_vendor = virt_to_phys_or_null(vendor); @@ -791,6 +805,8 @@ efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor, status = efi_thunk(get_variable, phys_name, phys_vendor, phys_attr, phys_data_size, phys_data); + spin_unlock_irqrestore(&efi_runtime_lock, flags); + return status; } @@ -800,6 +816,34 @@ efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor, { u32 phys_name, phys_vendor, phys_data; efi_status_t status; + unsigned long flags; + + spin_lock_irqsave(&efi_runtime_lock, flags); + + phys_name = virt_to_phys_or_null_size(name, efi_name_size(name)); + phys_vendor = virt_to_phys_or_null(vendor); + phys_data = virt_to_phys_or_null_size(data, data_size); + + /* If data_size is > sizeof(u32) we've got problems */ + status = efi_thunk(set_variable, phys_name, phys_vendor, + attr, data_size, phys_data); + + spin_unlock_irqrestore(&efi_runtime_lock, flags); + + return status; +} + +static efi_status_t +efi_thunk_set_variable_nonblocking(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, + void *data) +{ + u32 phys_name, phys_vendor, phys_data; + efi_status_t status; + unsigned long flags; + + if (!spin_trylock_irqsave(&efi_runtime_lock, flags)) + return EFI_NOT_READY; phys_name = virt_to_phys_or_null_size(name, efi_name_size(name)); phys_vendor = virt_to_phys_or_null(vendor); @@ -809,6 +853,8 @@ efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor, status = efi_thunk(set_variable, phys_name, phys_vendor, attr, data_size, phys_data); + spin_unlock_irqrestore(&efi_runtime_lock, flags); + return status; } @@ -819,6 +865,9 @@ efi_thunk_get_next_variable(unsigned long *name_size, { efi_status_t status; u32 phys_name_size, phys_name, phys_vendor; + unsigned long flags; + + spin_lock_irqsave(&efi_runtime_lock, flags); phys_name_size = virt_to_phys_or_null(name_size); phys_vendor = virt_to_phys_or_null(vendor); @@ -827,6 +876,8 @@ efi_thunk_get_next_variable(unsigned long *name_size, status = efi_thunk(get_next_variable, phys_name_size, phys_name, phys_vendor); + spin_unlock_irqrestore(&efi_runtime_lock, flags); + return status; } @@ -835,10 +886,15 @@ efi_thunk_get_next_high_mono_count(u32 *count) { efi_status_t status; u32 phys_count; + unsigned long flags; + + spin_lock_irqsave(&efi_runtime_lock, flags); phys_count = virt_to_phys_or_null(count); status = efi_thunk(get_next_high_mono_count, phys_count); + spin_unlock_irqrestore(&efi_runtime_lock, flags); + return status; } @@ -847,10 +903,15 @@ efi_thunk_reset_system(int reset_type, efi_status_t status, unsigned long data_size, efi_char16_t *data) { u32 phys_data; + unsigned long flags; + + spin_lock_irqsave(&efi_runtime_lock, flags); phys_data = virt_to_phys_or_null_size(data, data_size); efi_thunk(reset_system, reset_type, status, data_size, phys_data); + + spin_unlock_irqrestore(&efi_runtime_lock, flags); } static efi_status_t @@ -872,10 +933,40 @@ efi_thunk_query_variable_info(u32 attr, u64 *storage_space, { efi_status_t status; u32 phys_storage, phys_remaining, phys_max; + unsigned long flags; + + if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION) + return EFI_UNSUPPORTED; + + spin_lock_irqsave(&efi_runtime_lock, flags); + + phys_storage = virt_to_phys_or_null(storage_space); + phys_remaining = virt_to_phys_or_null(remaining_space); + phys_max = virt_to_phys_or_null(max_variable_size); + + status = efi_thunk(query_variable_info, attr, phys_storage, + phys_remaining, phys_max); + + spin_unlock_irqrestore(&efi_runtime_lock, flags); + + return status; +} + +static efi_status_t +efi_thunk_query_variable_info_nonblocking(u32 attr, u64 *storage_space, + u64 *remaining_space, + u64 *max_variable_size) +{ + efi_status_t status; + u32 phys_storage, phys_remaining, phys_max; + unsigned long flags; if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION) return EFI_UNSUPPORTED; + if (!spin_trylock_irqsave(&efi_runtime_lock, flags)) + return EFI_NOT_READY; + phys_storage = virt_to_phys_or_null(storage_space); phys_remaining = virt_to_phys_or_null(remaining_space); phys_max = virt_to_phys_or_null(max_variable_size); @@ -883,6 +974,8 @@ efi_thunk_query_variable_info(u32 attr, u64 *storage_space, status = efi_thunk(query_variable_info, attr, phys_storage, phys_remaining, phys_max); + spin_unlock_irqrestore(&efi_runtime_lock, flags); + return status; } @@ -908,9 +1001,11 @@ void efi_thunk_runtime_setup(void) efi.get_variable = efi_thunk_get_variable; efi.get_next_variable = efi_thunk_get_next_variable; efi.set_variable = efi_thunk_set_variable; + efi.set_variable_nonblocking = efi_thunk_set_variable_nonblocking; efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count; efi.reset_system = efi_thunk_reset_system; efi.query_variable_info = efi_thunk_query_variable_info; + efi.query_variable_info_nonblocking = efi_thunk_query_variable_info_nonblocking; efi.update_capsule = efi_thunk_update_capsule; efi.query_capsule_caps = efi_thunk_query_capsule_caps; } diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 36c1f8b9f7e0..844d31cb8a0c 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -105,12 +105,11 @@ early_param("efi_no_storage_paranoia", setup_storage_paranoia); */ void efi_delete_dummy_variable(void) { - efi.set_variable((efi_char16_t *)efi_dummy_name, - &EFI_DUMMY_GUID, - EFI_VARIABLE_NON_VOLATILE | - EFI_VARIABLE_BOOTSERVICE_ACCESS | - EFI_VARIABLE_RUNTIME_ACCESS, - 0, NULL); + efi.set_variable_nonblocking((efi_char16_t *)efi_dummy_name, + &EFI_DUMMY_GUID, + EFI_VARIABLE_NON_VOLATILE | + EFI_VARIABLE_BOOTSERVICE_ACCESS | + EFI_VARIABLE_RUNTIME_ACCESS, 0, NULL); } /* @@ -249,7 +248,8 @@ void __init efi_arch_mem_reserve(phys_addr_t addr, u64 size) int num_entries; void *new; - if (efi_mem_desc_lookup(addr, &md)) { + if (efi_mem_desc_lookup(addr, &md) || + md.type != EFI_BOOT_SERVICES_DATA) { pr_err("Failed to lookup EFI memory descriptor for %pa\n", &addr); return; } diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index fa021dfab088..5cf886c867c2 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,4 +1,4 @@ -obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o mfld.o mrfld.o pwr.o +obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o pwr.o # SFI specific code ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_wdt.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_wdt.c index 4f5fa65a1011..2acd6be13375 100644 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_wdt.c +++ b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_wdt.c @@ -18,6 +18,7 @@ #include <asm/intel-mid.h> #include <asm/intel_scu_ipc.h> #include <asm/io_apic.h> +#include <asm/hw_irq.h> #define TANGIER_EXT_TIMER0_MSI 12 diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 2ebdf31d9996..56f66eafb94f 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -36,8 +36,6 @@ #include <asm/apb_timer.h> #include <asm/reboot.h> -#include "intel_mid_weak_decls.h" - /* * the clockevent devices on Moorestown/Medfield can be APBT or LAPIC clock, * cmdline option x86_intel_mid_timer can be used to override the configuration @@ -61,10 +59,6 @@ enum intel_mid_timer_options intel_mid_timer_options; -/* intel_mid_ops to store sub arch ops */ -static struct intel_mid_ops *intel_mid_ops; -/* getter function for sub arch ops*/ -static void *(*get_intel_mid_ops[])(void) = INTEL_MID_OPS_INIT; enum intel_mid_cpu_type __intel_mid_cpu_chip; EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); @@ -82,11 +76,6 @@ static void intel_mid_reboot(void) intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0); } -static unsigned long __init intel_mid_calibrate_tsc(void) -{ - return 0; -} - static void __init intel_mid_setup_bp_timer(void) { apbt_time_init(); @@ -133,6 +122,7 @@ static void intel_mid_arch_setup(void) case 0x3C: case 0x4A: __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_TANGIER; + x86_platform.legacy.rtc = 1; break; case 0x27: default: @@ -140,17 +130,7 @@ static void intel_mid_arch_setup(void) break; } - if (__intel_mid_cpu_chip < MAX_CPU_OPS(get_intel_mid_ops)) - intel_mid_ops = get_intel_mid_ops[__intel_mid_cpu_chip](); - else { - intel_mid_ops = get_intel_mid_ops[INTEL_MID_CPU_CHIP_PENWELL](); - pr_info("ARCH: Unknown SoC, assuming Penwell!\n"); - } - out: - if (intel_mid_ops->arch_setup) - intel_mid_ops->arch_setup(); - /* * Intel MID platforms are using explicitly defined regulators. * @@ -191,7 +171,6 @@ void __init x86_intel_mid_early_setup(void) x86_cpuinit.setup_percpu_clockev = apbt_setup_secondary_clock; - x86_platform.calibrate_tsc = intel_mid_calibrate_tsc; x86_platform.get_nmi_reason = intel_mid_get_nmi_reason; x86_init.pci.arch_init = intel_mid_pci_init; diff --git a/arch/x86/platform/intel-mid/intel_mid_weak_decls.h b/arch/x86/platform/intel-mid/intel_mid_weak_decls.h deleted file mode 100644 index 3c1c3866d82b..000000000000 --- a/arch/x86/platform/intel-mid/intel_mid_weak_decls.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * intel_mid_weak_decls.h: Weak declarations of intel-mid.c - * - * (C) Copyright 2013 Intel Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. - */ - - -/* For every CPU addition a new get_<cpuname>_ops interface needs - * to be added. - */ -extern void *get_penwell_ops(void); -extern void *get_cloverview_ops(void); -extern void *get_tangier_ops(void); diff --git a/arch/x86/platform/intel-mid/mfld.c b/arch/x86/platform/intel-mid/mfld.c deleted file mode 100644 index e42978d4deaf..000000000000 --- a/arch/x86/platform/intel-mid/mfld.c +++ /dev/null @@ -1,70 +0,0 @@ -/* - * mfld.c: Intel Medfield platform setup code - * - * (C) Copyright 2013 Intel Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. - */ - -#include <linux/init.h> - -#include <asm/apic.h> -#include <asm/intel-mid.h> -#include <asm/intel_mid_vrtc.h> - -#include "intel_mid_weak_decls.h" - -static unsigned long __init mfld_calibrate_tsc(void) -{ - unsigned long fast_calibrate; - u32 lo, hi, ratio, fsb; - - rdmsr(MSR_IA32_PERF_STATUS, lo, hi); - pr_debug("IA32 perf status is 0x%x, 0x%0x\n", lo, hi); - ratio = (hi >> 8) & 0x1f; - pr_debug("ratio is %d\n", ratio); - if (!ratio) { - pr_err("read a zero ratio, should be incorrect!\n"); - pr_err("force tsc ratio to 16 ...\n"); - ratio = 16; - } - rdmsr(MSR_FSB_FREQ, lo, hi); - if ((lo & 0x7) == 0x7) - fsb = FSB_FREQ_83SKU; - else - fsb = FSB_FREQ_100SKU; - fast_calibrate = ratio * fsb; - pr_debug("read penwell tsc %lu khz\n", fast_calibrate); - lapic_timer_frequency = fsb * 1000 / HZ; - - /* - * TSC on Intel Atom SoCs is reliable and of known frequency. - * See tsc_msr.c for details. - */ - setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); - setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); - - return fast_calibrate; -} - -static void __init penwell_arch_setup(void) -{ - x86_platform.calibrate_tsc = mfld_calibrate_tsc; -} - -static struct intel_mid_ops penwell_ops = { - .arch_setup = penwell_arch_setup, -}; - -void *get_penwell_ops(void) -{ - return &penwell_ops; -} - -void *get_cloverview_ops(void) -{ - return &penwell_ops; -} diff --git a/arch/x86/platform/intel-mid/mrfld.c b/arch/x86/platform/intel-mid/mrfld.c deleted file mode 100644 index ae7bdeb0e507..000000000000 --- a/arch/x86/platform/intel-mid/mrfld.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Intel Merrifield platform specific setup code - * - * (C) Copyright 2013 Intel Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. - */ - -#include <linux/init.h> - -#include <asm/apic.h> -#include <asm/intel-mid.h> - -#include "intel_mid_weak_decls.h" - -static unsigned long __init tangier_calibrate_tsc(void) -{ - unsigned long fast_calibrate; - u32 lo, hi, ratio, fsb, bus_freq; - - /* *********************** */ - /* Compute TSC:Ratio * FSB */ - /* *********************** */ - - /* Compute Ratio */ - rdmsr(MSR_PLATFORM_INFO, lo, hi); - pr_debug("IA32 PLATFORM_INFO is 0x%x : %x\n", hi, lo); - - ratio = (lo >> 8) & 0xFF; - pr_debug("ratio is %d\n", ratio); - if (!ratio) { - pr_err("Read a zero ratio, force tsc ratio to 4 ...\n"); - ratio = 4; - } - - /* Compute FSB */ - rdmsr(MSR_FSB_FREQ, lo, hi); - pr_debug("Actual FSB frequency detected by SOC 0x%x : %x\n", - hi, lo); - - bus_freq = lo & 0x7; - pr_debug("bus_freq = 0x%x\n", bus_freq); - - if (bus_freq == 0) - fsb = FSB_FREQ_100SKU; - else if (bus_freq == 1) - fsb = FSB_FREQ_100SKU; - else if (bus_freq == 2) - fsb = FSB_FREQ_133SKU; - else if (bus_freq == 3) - fsb = FSB_FREQ_167SKU; - else if (bus_freq == 4) - fsb = FSB_FREQ_83SKU; - else if (bus_freq == 5) - fsb = FSB_FREQ_400SKU; - else if (bus_freq == 6) - fsb = FSB_FREQ_267SKU; - else if (bus_freq == 7) - fsb = FSB_FREQ_333SKU; - else { - BUG(); - pr_err("Invalid bus_freq! Setting to minimal value!\n"); - fsb = FSB_FREQ_100SKU; - } - - /* TSC = FSB Freq * Resolved HFM Ratio */ - fast_calibrate = ratio * fsb; - pr_debug("calculate tangier tsc %lu KHz\n", fast_calibrate); - - /* ************************************ */ - /* Calculate Local APIC Timer Frequency */ - /* ************************************ */ - lapic_timer_frequency = (fsb * 1000) / HZ; - - pr_debug("Setting lapic_timer_frequency = %d\n", - lapic_timer_frequency); - - /* - * TSC on Intel Atom SoCs is reliable and of known frequency. - * See tsc_msr.c for details. - */ - setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); - setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); - - return fast_calibrate; -} - -static void __init tangier_arch_setup(void) -{ - x86_platform.calibrate_tsc = tangier_calibrate_tsc; - x86_platform.legacy.rtc = 1; -} - -/* tangier arch ops */ -static struct intel_mid_ops tangier_ops = { - .arch_setup = tangier_arch_setup, -}; - -void *get_tangier_ops(void) -{ - return &tangier_ops; -} diff --git a/arch/x86/platform/olpc/olpc.c b/arch/x86/platform/olpc/olpc.c index 7c3077e58fa0..f0e920fb98ad 100644 --- a/arch/x86/platform/olpc/olpc.c +++ b/arch/x86/platform/olpc/olpc.c @@ -311,10 +311,8 @@ static int __init add_xo1_platform_devices(void) return PTR_ERR(pdev); pdev = platform_device_register_simple("olpc-xo1", -1, NULL, 0); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); - return 0; + return PTR_ERR_OR_ZERO(pdev); } static int olpc_xo1_ec_probe(struct platform_device *pdev) diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c index ca446da48fd2..a4130b84d1ff 100644 --- a/arch/x86/platform/uv/tlb_uv.c +++ b/arch/x86/platform/uv/tlb_uv.c @@ -1285,6 +1285,7 @@ void uv_bau_message_interrupt(struct pt_regs *regs) struct msg_desc msgdesc; ack_APIC_irq(); + kvm_set_cpu_l1tf_flush_l1d(); time_start = get_cycles(); bcp = &per_cpu(bau_control, smp_processor_id()); @@ -1607,8 +1608,6 @@ static int parse_tunables_write(struct bau_control *bcp, char *instr, *tunables[cnt].tunp = val; continue; } - if (q == p) - break; } return 0; } diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index ce8da3a0412c..fd369a6e9ff8 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -137,7 +137,7 @@ ENTRY(restore_registers) /* Saved in save_processor_state. */ lgdt saved_context_gdt_desc(%rax) - xorq %rax, %rax + xorl %eax, %eax /* tell the hibernation core that we've just restored the memory */ movq %rax, in_suspend(%rip) diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index 2e9ee023e6bc..81a8e33115ad 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -6,7 +6,7 @@ purgatory-y := purgatory.o stack.o setup-x86_$(BITS).o sha256.o entry64.o string targets += $(purgatory-y) PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) -$(obj)/sha256.o: $(srctree)/lib/sha256.c +$(obj)/sha256.o: $(srctree)/lib/sha256.c FORCE $(call if_changed_rule,cc_o_c) LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined -nostdlib -z nodefaultlib diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c index 220e97841e49..3a6c8ebc8032 100644 --- a/arch/x86/tools/relocs.c +++ b/arch/x86/tools/relocs.c @@ -67,6 +67,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { "__tracedata_(start|end)|" "__(start|stop)_notes|" "__end_rodata|" + "__end_rodata_aligned|" "__initramfs_start|" "(jiffies|jiffies_64)|" #if ELF_BITS == 64 diff --git a/arch/x86/um/mem_32.c b/arch/x86/um/mem_32.c index 744afdc18cf3..56c44d865f7b 100644 --- a/arch/x86/um/mem_32.c +++ b/arch/x86/um/mem_32.c @@ -16,7 +16,7 @@ static int __init gate_vma_init(void) if (!FIXADDR_USER_START) return 0; - gate_vma.vm_mm = NULL; + vma_init(&gate_vma, NULL); gate_vma.vm_start = FIXADDR_USER_START; gate_vma.vm_end = FIXADDR_USER_END; gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; diff --git a/arch/x86/um/vdso/.gitignore b/arch/x86/um/vdso/.gitignore index 9cac6d072199..f8b69d84238e 100644 --- a/arch/x86/um/vdso/.gitignore +++ b/arch/x86/um/vdso/.gitignore @@ -1,2 +1 @@ -vdso-syms.lds vdso.lds diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile index b2d6967262b2..822ccdba93ad 100644 --- a/arch/x86/um/vdso/Makefile +++ b/arch/x86/um/vdso/Makefile @@ -53,22 +53,6 @@ $(vobjs): KBUILD_CFLAGS += $(CFL) CFLAGS_REMOVE_vdso-note.o = -pg -fprofile-arcs -ftest-coverage CFLAGS_REMOVE_um_vdso.o = -pg -fprofile-arcs -ftest-coverage -targets += vdso-syms.lds -extra-$(VDSO64-y) += vdso-syms.lds - -# -# Match symbols in the DSO that look like VDSO*; produce a file of constants. -# -sed-vdsosym := -e 's/^00*/0/' \ - -e 's/^\([0-9a-fA-F]*\) . \(VDSO[a-zA-Z0-9_]*\)$$/\2 = 0x\1;/p' -quiet_cmd_vdsosym = VDSOSYM $@ -define cmd_vdsosym - $(NM) $< | LC_ALL=C sed -n $(sed-vdsosym) | LC_ALL=C sort > $@ -endef - -$(obj)/%-syms.lds: $(obj)/%.so.dbg FORCE - $(call if_changed,vdsosym) - # # The DSO images are built using a special linker script. # diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 3b5318505c69..2eeddd814653 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -3,6 +3,7 @@ #endif #include <linux/cpu.h> #include <linux/kexec.h> +#include <linux/slab.h> #include <xen/features.h> #include <xen/page.h> diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 8d4e2e1ae60b..105a57d73701 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -119,6 +119,27 @@ static void __init xen_banner(void) version >> 16, version & 0xffff, extra.extraversion, xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : ""); } + +static void __init xen_pv_init_platform(void) +{ + set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); + HYPERVISOR_shared_info = (void *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); + + /* xen clock uses per-cpu vcpu_info, need to init it for boot cpu */ + xen_vcpu_info_reset(0); + + /* pvclock is in shared info area */ + xen_init_time_ops(); +} + +static void __init xen_pv_guest_late_init(void) +{ +#ifndef CONFIG_SMP + /* Setup shared vcpu info for non-smp configurations */ + xen_setup_vcpu_info_placement(); +#endif +} + /* Check if running on Xen version (major, minor) or later */ bool xen_running_on_version_or_later(unsigned int major, unsigned int minor) @@ -947,34 +968,8 @@ static void xen_write_msr(unsigned int msr, unsigned low, unsigned high) xen_write_msr_safe(msr, low, high); } -void xen_setup_shared_info(void) -{ - set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); - - HYPERVISOR_shared_info = - (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); - - xen_setup_mfn_list_list(); - - if (system_state == SYSTEM_BOOTING) { -#ifndef CONFIG_SMP - /* - * In UP this is as good a place as any to set up shared info. - * Limit this to boot only, at restore vcpu setup is done via - * xen_vcpu_restore(). - */ - xen_setup_vcpu_info_placement(); -#endif - /* - * Now that shared info is set up we can start using routines - * that point to pvclock area. - */ - xen_init_time_ops(); - } -} - /* This is called once we have the cpu_possible_mask */ -void __ref xen_setup_vcpu_info_placement(void) +void __init xen_setup_vcpu_info_placement(void) { int cpu; @@ -1207,12 +1202,20 @@ asmlinkage __visible void __init xen_start_kernel(void) xen_setup_features(); - xen_setup_machphys_mapping(); - /* Install Xen paravirt ops */ pv_info = xen_info; pv_init_ops.patch = paravirt_patch_default; pv_cpu_ops = xen_cpu_ops; + xen_init_irq_ops(); + + /* + * Setup xen_vcpu early because it is needed for + * local_irq_disable(), irqs_disabled(), e.g. in printk(). + * + * Don't do the full vcpu_info placement stuff until we have + * the cpu_possible_mask and a non-dummy shared_info. + */ + xen_vcpu_info_reset(0); x86_platform.get_nmi_reason = xen_get_nmi_reason; @@ -1220,15 +1223,19 @@ asmlinkage __visible void __init xen_start_kernel(void) x86_init.irqs.intr_mode_init = x86_init_noop; x86_init.oem.arch_setup = xen_arch_setup; x86_init.oem.banner = xen_banner; + x86_init.hyper.init_platform = xen_pv_init_platform; + x86_init.hyper.guest_late_init = xen_pv_guest_late_init; /* * Set up some pagetable state before starting to set any ptes. */ + xen_setup_machphys_mapping(); xen_init_mmu_ops(); /* Prevent unwanted bits from being set in PTEs. */ __supported_pte_mask &= ~_PAGE_GLOBAL; + __default_kernel_pte_mask &= ~_PAGE_GLOBAL; /* * Prevent page tables from being allocated in highmem, even @@ -1249,20 +1256,9 @@ asmlinkage __visible void __init xen_start_kernel(void) get_cpu_cap(&boot_cpu_data); x86_configure_nx(); - xen_init_irq_ops(); - /* Let's presume PV guests always boot on vCPU with id 0. */ per_cpu(xen_vcpu_id, 0) = 0; - /* - * Setup xen_vcpu early because idt_setup_early_handler needs it for - * local_irq_disable(), irqs_disabled(). - * - * Don't do the full vcpu_info placement stuff until we have - * the cpu_possible_mask and a non-dummy shared_info. - */ - xen_vcpu_info_reset(0); - idt_setup_early_handler(); xen_init_capabilities(); diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index 74179852e46c..7515a19fd324 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -128,8 +128,6 @@ static const struct pv_irq_ops xen_irq_ops __initconst = { void __init xen_init_irq_ops(void) { - /* For PVH we use default pv_irq_ops settings. */ - if (!xen_feature(XENFEAT_hvm_callback_vector)) - pv_irq_ops = xen_irq_ops; + pv_irq_ops = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 2c30cabfda90..52206ad81e4b 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -1230,8 +1230,7 @@ static void __init xen_pagetable_p2m_free(void) * We roundup to the PMD, which means that if anybody at this stage is * using the __ka address of xen_start_info or * xen_start_info->shared_info they are in going to crash. Fortunatly - * we have already revectored in xen_setup_kernel_pagetable and in - * xen_setup_shared_info. + * we have already revectored in xen_setup_kernel_pagetable. */ size = roundup(size, PMD_SIZE); @@ -1292,8 +1291,7 @@ static void __init xen_pagetable_init(void) /* Remap memory freed due to conflicts with E820 map */ xen_remap_memory(); - - xen_setup_shared_info(); + xen_setup_mfn_list_list(); } static void xen_write_cr2(unsigned long cr2) { diff --git a/arch/x86/xen/suspend_pv.c b/arch/x86/xen/suspend_pv.c index a2e0f110af56..8303b58c79a9 100644 --- a/arch/x86/xen/suspend_pv.c +++ b/arch/x86/xen/suspend_pv.c @@ -27,8 +27,9 @@ void xen_pv_pre_suspend(void) void xen_pv_post_suspend(int suspend_cancelled) { xen_build_mfn_list_list(); - - xen_setup_shared_info(); + set_fixmap(FIX_PARAVIRT_BOOTMAP, xen_start_info->shared_info); + HYPERVISOR_shared_info = (void *)fix_to_virt(FIX_PARAVIRT_BOOTMAP); + xen_setup_mfn_list_list(); if (suspend_cancelled) { xen_start_info->store_mfn = diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index e0f1bcf01d63..c84f1e039d84 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -31,6 +31,8 @@ /* Xen may fire a timer up to this many ns early */ #define TIMER_SLOP 100000 +static u64 xen_sched_clock_offset __read_mostly; + /* Get the TSC speed from Xen */ static unsigned long xen_tsc_khz(void) { @@ -40,7 +42,7 @@ static unsigned long xen_tsc_khz(void) return pvclock_tsc_khz(info); } -u64 xen_clocksource_read(void) +static u64 xen_clocksource_read(void) { struct pvclock_vcpu_time_info *src; u64 ret; @@ -57,6 +59,11 @@ static u64 xen_clocksource_get_cycles(struct clocksource *cs) return xen_clocksource_read(); } +static u64 xen_sched_clock(void) +{ + return xen_clocksource_read() - xen_sched_clock_offset; +} + static void xen_read_wallclock(struct timespec64 *ts) { struct shared_info *s = HYPERVISOR_shared_info; @@ -367,7 +374,7 @@ void xen_timer_resume(void) } static const struct pv_time_ops xen_time_ops __initconst = { - .sched_clock = xen_clocksource_read, + .sched_clock = xen_sched_clock, .steal_clock = xen_steal_clock, }; @@ -503,8 +510,9 @@ static void __init xen_time_init(void) pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); } -void __ref xen_init_time_ops(void) +void __init xen_init_time_ops(void) { + xen_sched_clock_offset = xen_clocksource_read(); pv_time_ops = xen_time_ops; x86_init.timers.timer_init = xen_time_init; @@ -542,11 +550,11 @@ void __init xen_hvm_init_time_ops(void) return; if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { - printk(KERN_INFO "Xen doesn't support pvclock on HVM," - "disable pv timer\n"); + pr_info("Xen doesn't support pvclock on HVM, disable pv timer"); return; } + xen_sched_clock_offset = xen_clocksource_read(); pv_time_ops = xen_time_ops; x86_init.timers.setup_percpu_clockev = xen_time_init; x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 3b34745d0a52..e78684597f57 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -31,7 +31,6 @@ extern struct shared_info xen_dummy_shared_info; extern struct shared_info *HYPERVISOR_shared_info; void xen_setup_mfn_list_list(void); -void xen_setup_shared_info(void); void xen_build_mfn_list_list(void); void xen_setup_machphys_mapping(void); void xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn); @@ -68,12 +67,11 @@ void xen_init_irq_ops(void); void xen_setup_timer(int cpu); void xen_setup_runstate_info(int cpu); void xen_teardown_timer(int cpu); -u64 xen_clocksource_read(void); void xen_setup_cpu_clockevents(void); void xen_save_time_memory_area(void); void xen_restore_time_memory_area(void); -void __ref xen_init_time_ops(void); -void __init xen_hvm_init_time_ops(void); +void xen_init_time_ops(void); +void xen_hvm_init_time_ops(void); irqreturn_t xen_debug_interrupt(int irq, void *dev_id); diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index e7a23f2a519a..7de0149e1cf7 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -197,107 +197,9 @@ ATOMIC_OPS(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -/** - * atomic_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ -#define atomic_sub_and_test(i,v) (atomic_sub_return((i),(v)) == 0) - -/** - * atomic_inc - increment atomic variable - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1. - */ -#define atomic_inc(v) atomic_add(1,(v)) - -/** - * atomic_inc - increment atomic variable - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1. - */ -#define atomic_inc_return(v) atomic_add_return(1,(v)) - -/** - * atomic_dec - decrement atomic variable - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1. - */ -#define atomic_dec(v) atomic_sub(1,(v)) - -/** - * atomic_dec_return - decrement atomic variable - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1. - */ -#define atomic_dec_return(v) atomic_sub_return(1,(v)) - -/** - * atomic_dec_and_test - decrement and test - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ -#define atomic_dec_and_test(v) (atomic_sub_return(1,(v)) == 0) - -/** - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_add_return(1,(v)) == 0) - -/** - * atomic_add_negative - add and test if negative - * @v: pointer of type atomic_t - * @i: integer value to add - * - * Atomically adds @i to @v and returns true - * if the result is negative, or false when - * result is greater than or equal to zero. - */ -#define atomic_add_negative(i,v) (atomic_add_return((i),(v)) < 0) - #define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -/** - * __atomic_add_unless - add unless the number is a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as it was not @u. - * Returns the old value of @v. - */ -static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - #endif /* __KERNEL__ */ #endif /* _XTENSA_ATOMIC_H */ diff --git a/arch/xtensa/include/asm/hw_breakpoint.h b/arch/xtensa/include/asm/hw_breakpoint.h index dbe3053b284a..9f119c1ca0b5 100644 --- a/arch/xtensa/include/asm/hw_breakpoint.h +++ b/arch/xtensa/include/asm/hw_breakpoint.h @@ -30,13 +30,16 @@ struct arch_hw_breakpoint { u16 type; }; +struct perf_event_attr; struct perf_event; struct pt_regs; struct task_struct; int hw_breakpoint_slots(int type); -int arch_check_bp_in_kernelspace(struct perf_event *bp); -int arch_validate_hwbkpt_settings(struct perf_event *bp); +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw); int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data); diff --git a/arch/xtensa/kernel/hw_breakpoint.c b/arch/xtensa/kernel/hw_breakpoint.c index b35656ab7dbd..c2e387c19cda 100644 --- a/arch/xtensa/kernel/hw_breakpoint.c +++ b/arch/xtensa/kernel/hw_breakpoint.c @@ -33,14 +33,13 @@ int hw_breakpoint_slots(int type) } } -int arch_check_bp_in_kernelspace(struct perf_event *bp) +int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) { unsigned int len; unsigned long va; - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - va = info->address; - len = bp->attr.bp_len; + va = hw->address; + len = hw->len; return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); } @@ -48,50 +47,41 @@ int arch_check_bp_in_kernelspace(struct perf_event *bp) /* * Construct an arch_hw_breakpoint from a perf_event. */ -static int arch_build_bp_info(struct perf_event *bp) +int hw_breakpoint_arch_parse(struct perf_event *bp, + const struct perf_event_attr *attr, + struct arch_hw_breakpoint *hw) { - struct arch_hw_breakpoint *info = counter_arch_bp(bp); - /* Type */ - switch (bp->attr.bp_type) { + switch (attr->bp_type) { case HW_BREAKPOINT_X: - info->type = XTENSA_BREAKPOINT_EXECUTE; + hw->type = XTENSA_BREAKPOINT_EXECUTE; break; case HW_BREAKPOINT_R: - info->type = XTENSA_BREAKPOINT_LOAD; + hw->type = XTENSA_BREAKPOINT_LOAD; break; case HW_BREAKPOINT_W: - info->type = XTENSA_BREAKPOINT_STORE; + hw->type = XTENSA_BREAKPOINT_STORE; break; case HW_BREAKPOINT_RW: - info->type = XTENSA_BREAKPOINT_LOAD | XTENSA_BREAKPOINT_STORE; + hw->type = XTENSA_BREAKPOINT_LOAD | XTENSA_BREAKPOINT_STORE; break; default: return -EINVAL; } /* Len */ - info->len = bp->attr.bp_len; - if (info->len < 1 || info->len > 64 || !is_power_of_2(info->len)) + hw->len = attr->bp_len; + if (hw->len < 1 || hw->len > 64 || !is_power_of_2(hw->len)) return -EINVAL; /* Address */ - info->address = bp->attr.bp_addr; - if (info->address & (info->len - 1)) + hw->address = attr->bp_addr; + if (hw->address & (hw->len - 1)) return -EINVAL; return 0; } -int arch_validate_hwbkpt_settings(struct perf_event *bp) -{ - int ret; - - /* Build the arch_hw_breakpoint. */ - ret = arch_build_bp_info(bp); - return ret; -} - int hw_breakpoint_exceptions_notify(struct notifier_block *unused, unsigned long val, void *data) { |