diff options
Diffstat (limited to 'arch/arm')
34 files changed, 311 insertions, 413 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e00d94b16658..3996b6572c3a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -49,6 +49,7 @@ config ARM select GENERIC_ARCH_TOPOLOGY if ARM_CPU_TOPOLOGY select GENERIC_ATOMIC64 if CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI select GENERIC_CLOCKEVENTS_BROADCAST if SMP + select GENERIC_IRQ_IPI if SMP select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP @@ -67,6 +68,7 @@ config ARM select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU select HAVE_ARCH_MMAP_RND_BITS if MMU + select HAVE_ARCH_SECCOMP select HAVE_ARCH_SECCOMP_FILTER if AEABI && !OABI_COMPAT select HAVE_ARCH_THREAD_STRUCT_WHITELIST select HAVE_ARCH_TRACEHOOK @@ -83,7 +85,7 @@ config ARM select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && (CC_IS_GCC || CLANG_VERSION >= 100000) + select HAVE_FUNCTION_TRACER if !XIP_KERNEL select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT if PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7) select HAVE_IDE if PCI || ISA || PCMCIA @@ -1617,20 +1619,6 @@ config UACCESS_WITH_MEMCPY However, if the CPU data cache is using a write-allocate mode, this option is unlikely to provide any performance gain. -config SECCOMP - bool - prompt "Enable seccomp to safely compute untrusted bytecode" - help - This kernel feature is useful for number crunching applications - that may need to compute untrusted bytecode during their - execution. By using pipes or other transports made available to - the process as file descriptors supporting the read/write - syscalls, it's possible to isolate those applications in - their own address space using seccomp. Once seccomp is - enabled via prctl(PR_SET_SECCOMP), it cannot be disabled - and the task is only allowed to execute a few safe syscalls - defined by each seccomp mode. - config PARAVIRT bool "Enable paravirtualization code" help diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 4e877354515f..e589da3c8949 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -16,6 +16,10 @@ LDFLAGS_vmlinux += --be8 KBUILD_LDFLAGS_MODULE += --be8 endif +# We never want expected sections to be placed heuristically by the +# linker. All sections should be explicitly named in the linker script. +LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn) + ifeq ($(CONFIG_ARM_MODULE_PLTS),y) KBUILD_LDS_MODULE += $(srctree)/arch/arm/kernel/module.lds endif diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index b1147b7f2c8d..58028abd05d9 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -123,6 +123,8 @@ endif LDFLAGS_vmlinux += --no-undefined # Delete all temporary local symbols LDFLAGS_vmlinux += -X +# Report orphan sections +LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn) # Next argument is a linker script LDFLAGS_vmlinux += -T diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S index 09ac33f52814..b914be3a207b 100644 --- a/arch/arm/boot/compressed/vmlinux.lds.S +++ b/arch/arm/boot/compressed/vmlinux.lds.S @@ -2,6 +2,7 @@ /* * Copyright (C) 2000 Russell King */ +#include <asm/vmlinux.lds.h> #ifdef CONFIG_CPU_ENDIAN_BE8 #define ZIMAGE_MAGIC(x) ( (((x) >> 24) & 0x000000ff) | \ @@ -17,8 +18,11 @@ ENTRY(_start) SECTIONS { /DISCARD/ : { + COMMON_DISCARDS *(.ARM.exidx*) *(.ARM.extab*) + *(.note.*) + *(.rel.*) /* * Discard any r/w data - this produces a link error if we have any, * which is required for PIC decompression. Local data generates @@ -36,9 +40,7 @@ SECTIONS *(.start) *(.text) *(.text.*) - *(.gnu.warning) - *(.glue_7t) - *(.glue_7) + ARM_STUBS_TEXT } .table : ALIGN(4) { _table_start = .; @@ -128,12 +130,10 @@ SECTIONS PROVIDE(__pecoff_data_size = ALIGN(512) - ADDR(.data)); PROVIDE(__pecoff_end = ALIGN(512)); - .stab 0 : { *(.stab) } - .stabstr 0 : { *(.stabstr) } - .stab.excl 0 : { *(.stab.excl) } - .stab.exclstr 0 : { *(.stab.exclstr) } - .stab.index 0 : { *(.stab.index) } - .stab.indexstr 0 : { *(.stab.indexstr) } - .comment 0 : { *(.comment) } + STABS_DEBUG + DWARF_DEBUG + ARM_DETAILS + + ARM_ASSERTS } ASSERT(_edata_real == _edata, "error: zImage file size is incorrect"); diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S index cfaed4e67535..7d0cc7f226a5 100644 --- a/arch/arm/crypto/aes-neonbs-core.S +++ b/arch/arm/crypto/aes-neonbs-core.S @@ -77,11 +77,6 @@ vldr \out\()h, \sym + 8 .endm - .macro __adr, reg, lbl - adr \reg, \lbl -THUMB( orr \reg, \reg, #1 ) - .endm - .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7 veor \b2, \b2, \b1 veor \b5, \b5, \b6 @@ -629,11 +624,11 @@ ENDPROC(aesbs_decrypt8) push {r4-r6, lr} ldr r5, [sp, #16] // number of blocks -99: __adr ip, 0f +99: adr ip, 0f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [r1]! vld1.8 {q1}, [r1]! @@ -648,11 +643,11 @@ ENDPROC(aesbs_decrypt8) mov rounds, r3 bl \do8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vst1.8 {\o0}, [r0]! vst1.8 {\o1}, [r0]! @@ -689,12 +684,12 @@ ENTRY(aesbs_cbc_decrypt) push {r4-r6, lr} ldm ip, {r5-r6} // load args 4-5 -99: __adr ip, 0f +99: adr ip, 0f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 mov lr, r1 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [lr]! vld1.8 {q1}, [lr]! @@ -718,11 +713,11 @@ ENTRY(aesbs_cbc_decrypt) vmov q14, q8 vmov q15, q8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q9}, [r1]! vld1.8 {q10}, [r1]! @@ -733,9 +728,9 @@ ENTRY(aesbs_cbc_decrypt) vld1.8 {q15}, [r1]! W(nop) -1: __adr ip, 2f +1: adr ip, 2f sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor q0, q0, q8 vst1.8 {q0}, [r0]! @@ -804,13 +799,13 @@ ENTRY(aesbs_ctr_encrypt) vmov q6, q0 vmov q7, q0 - __adr ip, 0f + adr ip, 0f sub lr, r5, #1 and lr, lr, #7 cmp r5, #8 sub ip, ip, lr, lsl #5 sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 next_ctr q1 next_ctr q2 @@ -824,13 +819,13 @@ ENTRY(aesbs_ctr_encrypt) mov rounds, r3 bl aesbs_encrypt8 - __adr ip, 1f + adr ip, 1f and lr, r5, #7 cmp r5, #8 movgt r4, #0 ldrle r4, [sp, #40] // load final in the last round sub ip, ip, lr, lsl #2 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q8}, [r1]! vld1.8 {q9}, [r1]! @@ -843,10 +838,10 @@ ENTRY(aesbs_ctr_encrypt) 1: bne 2f vld1.8 {q15}, [r1]! -2: __adr ip, 3f +2: adr ip, 3f cmp r5, #8 sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor q0, q0, q8 vst1.8 {q0}, [r0]! @@ -900,12 +895,12 @@ __xts_prepare8: vshr.u64 d30, d31, #7 vmov q12, q14 - __adr ip, 0f + adr ip, 0f and r4, r6, #7 cmp r6, #8 sub ip, ip, r4, lsl #5 mov r4, sp - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q0}, [r1]! next_tweak q12, q14, q15, q13 @@ -961,8 +956,7 @@ ENDPROC(__xts_prepare8) push {r4-r8, lr} mov r5, sp // preserve sp ldrd r6, r7, [sp, #24] // get blocks and iv args - ldr r8, [sp, #32] // reorder final tweak? - rsb r8, r8, #1 + rsb r8, ip, #1 sub ip, sp, #128 // make room for 8x tweak bic ip, ip, #0xf // align sp to 16 bytes mov sp, ip @@ -973,12 +967,12 @@ ENDPROC(__xts_prepare8) mov rounds, r3 bl \do8 - __adr ip, 0f + adr ip, 0f and lr, r6, #7 cmp r6, #8 sub ip, ip, lr, lsl #2 mov r4, sp - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 vld1.8 {q8}, [r4, :128]! vld1.8 {q9}, [r4, :128]! @@ -989,9 +983,9 @@ ENDPROC(__xts_prepare8) vld1.8 {q14}, [r4, :128]! vld1.8 {q15}, [r4, :128] -0: __adr ip, 1f +0: adr ip, 1f sub ip, ip, lr, lsl #3 - bxlt ip // computed goto if blocks < 8 + movlt pc, ip // computed goto if blocks < 8 veor \o0, \o0, q8 vst1.8 {\o0}, [r0]! @@ -1018,9 +1012,11 @@ ENDPROC(__xts_prepare8) .endm ENTRY(aesbs_xts_encrypt) + mov ip, #0 // never reorder final tweak __xts_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5 ENDPROC(aesbs_xts_encrypt) ENTRY(aesbs_xts_decrypt) + ldr ip, [sp, #8] // reorder final tweak? __xts_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5 ENDPROC(aesbs_xts_decrypt) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index e6fd32919c81..bda8bf17631e 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -8,7 +8,6 @@ #include <asm/neon.h> #include <asm/simd.h> #include <crypto/aes.h> -#include <crypto/cbc.h> #include <crypto/ctr.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> @@ -49,7 +48,7 @@ struct aesbs_ctx { struct aesbs_cbc_ctx { struct aesbs_ctx key; - struct crypto_cipher *enc_tfm; + struct crypto_skcipher *enc_tfm; }; struct aesbs_xts_ctx { @@ -140,19 +139,23 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key, kernel_neon_end(); memzero_explicit(&rk, sizeof(rk)); - return crypto_cipher_setkey(ctx->enc_tfm, in_key, key_len); + return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len); } -static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) +static int cbc_encrypt(struct skcipher_request *req) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - crypto_cipher_encrypt_one(ctx->enc_tfm, dst, src); -} + skcipher_request_set_tfm(subreq, ctx->enc_tfm); + skcipher_request_set_callback(subreq, + skcipher_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); -static int cbc_encrypt(struct skcipher_request *req) -{ - return crypto_cbc_encrypt_walk(req, cbc_encrypt_one); + return crypto_skcipher_encrypt(subreq); } static int cbc_decrypt(struct skcipher_request *req) @@ -183,20 +186,27 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int cbc_init(struct crypto_tfm *tfm) +static int cbc_init(struct crypto_skcipher *tfm) { - struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + unsigned int reqsize; - ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0); + ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->enc_tfm)) + return PTR_ERR(ctx->enc_tfm); - return PTR_ERR_OR_ZERO(ctx->enc_tfm); + reqsize = sizeof(struct skcipher_request); + reqsize += crypto_skcipher_reqsize(ctx->enc_tfm); + crypto_skcipher_set_reqsize(tfm, reqsize); + + return 0; } -static void cbc_exit(struct crypto_tfm *tfm) +static void cbc_exit(struct crypto_skcipher *tfm) { - struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - crypto_free_cipher(ctx->enc_tfm); + crypto_free_skcipher(ctx->enc_tfm); } static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, @@ -304,9 +314,9 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return aesbs_setkey(tfm, in_key, key_len); } -static int xts_init(struct crypto_tfm *tfm) +static int xts_init(struct crypto_skcipher *tfm) { - struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0); if (IS_ERR(ctx->cts_tfm)) @@ -319,9 +329,9 @@ static int xts_init(struct crypto_tfm *tfm) return PTR_ERR_OR_ZERO(ctx->tweak_tfm); } -static void xts_exit(struct crypto_tfm *tfm) +static void xts_exit(struct crypto_skcipher *tfm) { - struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); crypto_free_cipher(ctx->tweak_tfm); crypto_free_cipher(ctx->cts_tfm); @@ -432,8 +442,6 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_init = cbc_init, - .base.cra_exit = cbc_exit, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -442,6 +450,8 @@ static struct skcipher_alg aes_algs[] = { { .setkey = aesbs_cbc_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, + .init = cbc_init, + .exit = cbc_exit, }, { .base.cra_name = "__ctr(aes)", .base.cra_driver_name = "__ctr-aes-neonbs", @@ -483,8 +493,6 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_init = xts_init, - .base.cra_exit = xts_exit, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -493,6 +501,8 @@ static struct skcipher_alg aes_algs[] = { { .setkey = aesbs_xts_setkey, .encrypt = xts_encrypt, .decrypt = xts_decrypt, + .init = xts_init, + .exit = xts_exit, } }; static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; diff --git a/arch/arm/crypto/curve25519-glue.c b/arch/arm/crypto/curve25519-glue.c index 776ae07e0469..31eb75b6002f 100644 --- a/arch/arm/crypto/curve25519-glue.c +++ b/arch/arm/crypto/curve25519-glue.c @@ -16,6 +16,7 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/jump_label.h> +#include <linux/scatterlist.h> #include <crypto/curve25519.h> asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 13cfef4ae22e..3023c1acfa19 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -20,6 +20,7 @@ void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); +void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce); void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl index 9f96ff48e4a8..f3a2b54efd4e 100644 --- a/arch/arm/crypto/sha256-armv4.pl +++ b/arch/arm/crypto/sha256-armv4.pl @@ -175,7 +175,6 @@ $code=<<___; #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -471,7 +470,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub $H,sp,#16*4+16 - adrl $Ktbl,K256 + adr $Ktbl,.Lsha256_block_data_order + sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped index ea04b2ab0c33..6363014a50d7 100644 --- a/arch/arm/crypto/sha256-core.S_shipped +++ b/arch/arm/crypto/sha256-core.S_shipped @@ -56,7 +56,6 @@ #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -1885,7 +1884,8 @@ sha256_block_data_order_neon: stmdb sp!,{r4-r12,lr} sub r11,sp,#16*4+16 - adrl r14,K256 + adr r14,.Lsha256_block_data_order + sub r14,r14,#.Lsha256_block_data_order-K256 bic r11,r11,#15 @ align for 128-bit stores mov r12,sp mov sp,r11 @ alloca diff --git a/arch/arm/crypto/sha512-armv4.pl b/arch/arm/crypto/sha512-armv4.pl index 69df68981acd..2fc3516912fa 100644 --- a/arch/arm/crypto/sha512-armv4.pl +++ b/arch/arm/crypto/sha512-armv4.pl @@ -212,7 +212,6 @@ $code=<<___; #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -602,7 +601,8 @@ sha512_block_data_order_neon: dmb @ errata #451034 on early Cortex A8 add $len,$inp,$len,lsl#7 @ len to point at the end of inp VFP_ABI_PUSH - adrl $Ktbl,K512 + adr $Ktbl,.Lsha512_block_data_order + sub $Ktbl,$Ktbl,.Lsha512_block_data_order-K512 vldmia $ctx,{$A-$H} @ load context .Loop_neon: ___ diff --git a/arch/arm/crypto/sha512-core.S_shipped b/arch/arm/crypto/sha512-core.S_shipped index cb147db5cbfe..03014624f2ab 100644 --- a/arch/arm/crypto/sha512-core.S_shipped +++ b/arch/arm/crypto/sha512-core.S_shipped @@ -79,7 +79,6 @@ #else .syntax unified # ifdef __thumb2__ -# define adrl adr .thumb # else .code 32 @@ -543,7 +542,8 @@ sha512_block_data_order_neon: dmb @ errata #451034 on early Cortex A8 add r2,r1,r2,lsl#7 @ len to point at the end of inp VFP_ABI_PUSH - adrl r3,K512 + adr r3,.Lsha512_block_data_order + sub r3,r3,.Lsha512_block_data_order-K512 vldmia r0,{d16-d23} @ load context .Loop_neon: vshr.u64 d24,d20,#14 @ 0 diff --git a/arch/arm/include/asm/checksum.h b/arch/arm/include/asm/checksum.h index ed6073fee338..f0f54aef3724 100644 --- a/arch/arm/include/asm/checksum.h +++ b/arch/arm/include/asm/checksum.h @@ -35,23 +35,20 @@ __wsum csum_partial(const void *buff, int len, __wsum sum); */ __wsum -csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum); +csum_partial_copy_nocheck(const void *src, void *dst, int len); __wsum -csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr); +csum_partial_copy_from_user(const void __user *src, void *dst, int len); #define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER +#define _HAVE_ARCH_CSUM_AND_COPY static inline -__wsum csum_and_copy_from_user (const void __user *src, void *dst, - int len, __wsum sum, int *err_ptr) +__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len) { - if (access_ok(src, len)) - return csum_partial_copy_from_user(src, dst, len, sum, err_ptr); + if (!access_ok(src, len)) + return 0; - if (len) - *err_ptr = -EFAULT; - - return sum; + return csum_partial_copy_from_user(src, dst, len); } /* diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 5dcf3c6011b7..3ee4f4381985 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -66,25 +66,24 @@ static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt) #define MAX_UNCOMP_KERNEL_SIZE SZ_32M /* - * The kernel zImage should preferably be located between 32 MB and 128 MB - * from the base of DRAM. The min address leaves space for a maximal size - * uncompressed image, and the max address is due to how the zImage decompressor - * picks a destination address. + * phys-to-virt patching requires that the physical to virtual offset fits + * into the immediate field of an add/sub instruction, which comes down to the + * 24 least significant bits being zero, and so the offset should be a multiple + * of 16 MB. Since PAGE_OFFSET itself is a multiple of 16 MB, the physical + * base should be aligned to 16 MB as well. */ -#define ZIMAGE_OFFSET_LIMIT SZ_128M -#define MIN_ZIMAGE_OFFSET MAX_UNCOMP_KERNEL_SIZE +#define EFI_PHYS_ALIGN SZ_16M -/* on ARM, the FDT should be located in the first 128 MB of RAM */ -static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base) +/* on ARM, the FDT should be located in a lowmem region */ +static inline unsigned long efi_get_max_fdt_addr(unsigned long image_addr) { - return dram_base + ZIMAGE_OFFSET_LIMIT; + return round_down(image_addr, EFI_PHYS_ALIGN) + SZ_512M; } /* on ARM, the initrd should be loaded in a lowmem region */ -static inline unsigned long efi_get_max_initrd_addr(unsigned long dram_base, - unsigned long image_addr) +static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) { - return dram_base + SZ_512M; + return round_down(image_addr, EFI_PHYS_ALIGN) + SZ_512M; } struct efi_arm_entry_state { diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 7a88f160b1fb..b95848ed2bc7 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -6,29 +6,12 @@ #include <linux/threads.h> #include <asm/irq.h> -/* number of IPIS _not_ including IPI_CPU_BACKTRACE */ -#define NR_IPI 7 - typedef struct { unsigned int __softirq_pending; -#ifdef CONFIG_SMP - unsigned int ipi_irqs[NR_IPI]; -#endif } ____cacheline_aligned irq_cpustat_t; #include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */ -#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ -#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) - -#ifdef CONFIG_SMP -u64 smp_irq_stat_cpu(unsigned int cpu); -#else -#define smp_irq_stat_cpu(cpu) 0 -#endif - -#define arch_irq_stat_cpu smp_irq_stat_cpu - #define __ARCH_IRQ_EXIT_IRQS_DISABLED 1 #endif /* __ASM_HARDIRQ_H */ diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index a91f21e3c5b5..0ca55a607d0a 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -39,11 +39,10 @@ void handle_IPI(int ipinr, struct pt_regs *regs); */ extern void smp_init_cpus(void); - /* - * Provide a function to raise an IPI cross call on CPUs in callmap. + * Register IPI interrupts with the arch SMP code */ -extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); +extern void set_smp_ipi_range(int ipi_base, int nr_ipi); /* * Called from platform specific assembly code, this is the diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index 9415222b49ad..b8cbe03ad260 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -59,6 +59,7 @@ __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) #ifdef CONFIG_ARM_LPAE struct page *page = virt_to_page(pmdp); + pgtable_pmd_page_dtor(page); tlb_remove_table(tlb, page); #endif } diff --git a/arch/arm/kernel/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index 381a8e105fa5..4a91428c324d 100644 --- a/arch/arm/kernel/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -1,4 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include <asm-generic/vmlinux.lds.h> #ifdef CONFIG_HOTPLUG_CPU #define ARM_CPU_DISCARD(x) @@ -49,8 +50,29 @@ EXIT_CALL \ ARM_MMU_DISCARD(*(.text.fixup)) \ ARM_MMU_DISCARD(*(__ex_table)) \ - *(.discard) \ - *(.discard.*) + COMMON_DISCARDS + +/* + * Sections that should stay zero sized, which is safer to explicitly + * check instead of blindly discarding. + */ +#define ARM_ASSERTS \ + .plt : { \ + *(.iplt) *(.rel.iplt) *(.iplt) *(.igot.plt) \ + } \ + ASSERT(SIZEOF(.plt) == 0, \ + "Unexpected run-time procedure linkages detected!") + +#define ARM_DETAILS \ + ELF_DETAILS \ + .ARM.attributes 0 : { *(.ARM.attributes) } + +#define ARM_STUBS_TEXT \ + *(.gnu.warning) \ + *(.glue_7) \ + *(.glue_7t) \ + *(.vfp11_veneer) \ + *(.v4_bx) #define ARM_TEXT \ IDMAP_TEXT \ @@ -64,9 +86,7 @@ CPUIDLE_TEXT \ LOCK_TEXT \ KPROBES_TEXT \ - *(.gnu.warning) \ - *(.glue_7) \ - *(.glue_7t) \ + ARM_STUBS_TEXT \ . = ALIGN(4); \ *(.got) /* Global offset table */ \ ARM_CPU_KEEP(PROC_INFO) diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index ee514034c0a1..698b6f636156 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -18,7 +18,6 @@ * IRQ's are in fact implemented a bit like signal handlers for the kernel. * Naturally it's not a 1:1 relation, but there are similarities. */ -#include <linux/kernel_stat.h> #include <linux/signal.h> #include <linux/ioport.h> #include <linux/interrupt.h> diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index d8e18cdd96d3..3f65d0ac9f63 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -843,20 +843,26 @@ early_param("mem", early_mem); static void __init request_standard_resources(const struct machine_desc *mdesc) { - struct memblock_region *region; + phys_addr_t start, end, res_end; struct resource *res; + u64 i; kernel_code.start = virt_to_phys(_text); kernel_code.end = virt_to_phys(__init_begin - 1); kernel_data.start = virt_to_phys(_sdata); kernel_data.end = virt_to_phys(_end - 1); - for_each_memblock(memory, region) { - phys_addr_t start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); - phys_addr_t end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + for_each_mem_range(i, &start, &end) { unsigned long boot_alias_start; /* + * In memblock, end points to the first byte after the + * range while in resourses, end points to the last byte in + * the range. + */ + res_end = end - 1; + + /* * Some systems have a special memory alias which is only * used for booting. We need to advertise this region to * kexec-tools so they know where bootable RAM is located. @@ -869,7 +875,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) __func__, sizeof(*res)); res->name = "System RAM (boot alias)"; res->start = boot_alias_start; - res->end = phys_to_idmap(end); + res->end = phys_to_idmap(res_end); res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; request_resource(&iomem_resource, res); } @@ -880,7 +886,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) sizeof(*res)); res->name = "System RAM"; res->start = start; - res->end = end; + res->end = res_end; res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; request_resource(&iomem_resource, res); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5d9da61eff62..48099c6e1e4a 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -26,6 +26,7 @@ #include <linux/completion.h> #include <linux/cpufreq.h> #include <linux/irq_work.h> +#include <linux/kernel_stat.h> #include <linux/atomic.h> #include <asm/bugs.h> @@ -65,18 +66,26 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_IRQ_WORK, IPI_COMPLETION, + NR_IPI, /* * CPU_BACKTRACE is special and not included in NR_IPI * or tracable with trace_ipi_* */ - IPI_CPU_BACKTRACE, + IPI_CPU_BACKTRACE = NR_IPI, /* * SGI8-15 can be reserved by secure firmware, and thus may * not be usable by the kernel. Please keep the above limited * to at most 8 entries. */ + MAX_IPI }; +static int ipi_irq_base __read_mostly; +static int nr_ipi __read_mostly = NR_IPI; +static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly; + +static void ipi_setup(int cpu); + static DECLARE_COMPLETION(cpu_running); static struct smp_operations smp_ops __ro_after_init; @@ -226,6 +235,17 @@ int platform_can_hotplug_cpu(unsigned int cpu) return cpu != 0; } +static void ipi_teardown(int cpu) +{ + int i; + + if (WARN_ON_ONCE(!ipi_irq_base)) + return; + + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_irq_base + i); +} + /* * __cpu_disable runs on the processor to be shutdown. */ @@ -247,6 +267,7 @@ int __cpu_disable(void) * and we must not schedule until we're ready to give up the cpu. */ set_cpu_online(cpu, false); + ipi_teardown(cpu); /* * OK - migrate IRQs away from this CPU @@ -422,6 +443,8 @@ asmlinkage void secondary_start_kernel(void) notify_cpu_starting(cpu); + ipi_setup(cpu); + calibrate_delay(); smp_store_cpu_info(cpu); @@ -500,14 +523,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -static void (*__smp_cross_call)(const struct cpumask *, unsigned int); - -void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) -{ - if (!__smp_cross_call) - __smp_cross_call = fn; -} - static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_WAKEUP, "CPU wakeup interrupts"), @@ -519,38 +534,28 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { S(IPI_COMPLETION, "completion interrupts"), }; -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) -{ - trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); - __smp_cross_call(target, ipinr); -} +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); void show_ipi_list(struct seq_file *p, int prec) { unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { + unsigned int irq; + + if (!ipi_desc[i]) + continue; + + irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", - __get_irq_stat(cpu, ipi_irqs[i])); + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); seq_printf(p, " %s\n", ipi_types[i]); } } -u64 smp_irq_stat_cpu(unsigned int cpu) -{ - u64 sum = 0; - int i; - - for (i = 0; i < NR_IPI; i++) - sum += __get_irq_stat(cpu, ipi_irqs[i]); - - return sum; -} - void arch_send_call_function_ipi_mask(const struct cpumask *mask) { smp_cross_call(mask, IPI_CALL_FUNC); @@ -627,15 +632,12 @@ asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) handle_IPI(ipinr, regs); } -void handle_IPI(int ipinr, struct pt_regs *regs) +static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - struct pt_regs *old_regs = set_irq_regs(regs); - if ((unsigned)ipinr < NR_IPI) { + if ((unsigned)ipinr < NR_IPI) trace_ipi_entry_rcuidle(ipi_types[ipinr]); - __inc_irq_stat(cpu, ipi_irqs[ipinr]); - } switch (ipinr) { case IPI_WAKEUP: @@ -643,9 +645,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST case IPI_TIMER: - irq_enter(); tick_receive_broadcast(); - irq_exit(); break; #endif @@ -654,36 +654,26 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); break; case IPI_CPU_STOP: - irq_enter(); ipi_cpu_stop(cpu); - irq_exit(); break; #ifdef CONFIG_IRQ_WORK case IPI_IRQ_WORK: - irq_enter(); irq_work_run(); - irq_exit(); break; #endif case IPI_COMPLETION: - irq_enter(); ipi_complete(cpu); - irq_exit(); break; case IPI_CPU_BACKTRACE: printk_nmi_enter(); - irq_enter(); - nmi_cpu_backtrace(regs); - irq_exit(); + nmi_cpu_backtrace(get_irq_regs()); printk_nmi_exit(); break; @@ -695,9 +685,67 @@ void handle_IPI(int ipinr, struct pt_regs *regs) if ((unsigned)ipinr < NR_IPI) trace_ipi_exit_rcuidle(ipi_types[ipinr]); +} + +/* Legacy version, should go away once all irqchips have been converted */ +void handle_IPI(int ipinr, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); + do_handle_IPI(ipinr); + irq_exit(); + set_irq_regs(old_regs); } +static irqreturn_t ipi_handler(int irq, void *data) +{ + do_handle_IPI(irq - ipi_irq_base); + return IRQ_HANDLED; +} + +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) +{ + trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); + __ipi_send_mask(ipi_desc[ipinr], target); +} + +static void ipi_setup(int cpu) +{ + int i; + + if (WARN_ON_ONCE(!ipi_irq_base)) + return; + + for (i = 0; i < nr_ipi; i++) + enable_percpu_irq(ipi_irq_base + i, 0); +} + +void __init set_smp_ipi_range(int ipi_base, int n) +{ + int i; + + WARN_ON(n < MAX_IPI); + nr_ipi = min(n, MAX_IPI); + + for (i = 0; i < nr_ipi; i++) { + int err; + + err = request_percpu_irq(ipi_base + i, ipi_handler, + "IPI", &irq_stat); + WARN_ON(err); + + ipi_desc[i] = irq_to_desc(ipi_base + i); + irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); + } + + ipi_irq_base = ipi_base; + + /* Setup the boot CPU immediately */ + ipi_setup(smp_processor_id()); +} + void smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); @@ -805,7 +853,7 @@ core_initcall(register_cpufreq_notifier); static void raise_nmi(cpumask_t *mask) { - __smp_cross_call(mask, IPI_CPU_BACKTRACE); + __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index b5adaf744630..ef0058de432b 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -178,15 +178,6 @@ static inline void update_cpu_capacity(unsigned int cpuid) {} #endif /* - * The current assumption is that we can power gate each core independently. - * This will be superseded by DT binding once available. - */ -const struct cpumask *cpu_corepower_mask(int cpu) -{ - return &cpu_topology[cpu].thread_sibling; -} - -/* * store_cpu_topology is called at boot when only one cpu is running * and with the mutex cpu_hotplug.lock locked, when several cpus have booted, * which prevents simultaneous write access to cpu_topology array @@ -241,20 +232,6 @@ topology_populated: update_siblings_masks(cpuid); } -static inline int cpu_corepower_flags(void) -{ - return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN; -} - -static struct sched_domain_topology_level arm_topology[] = { -#ifdef CONFIG_SCHED_MC - { cpu_corepower_mask, cpu_corepower_flags, SD_INIT_NAME(GMC) }, - { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, -#endif - { cpu_cpu_mask, SD_INIT_NAME(DIE) }, - { NULL, }, -}; - /* * init_cpu_topology is called at boot when only one cpu is running * which prevent simultaneous write access to cpu_topology array @@ -265,7 +242,4 @@ void __init init_cpu_topology(void) smp_wmb(); parse_dt_topology(); - - /* Set scheduler topology descriptor */ - set_sched_topology(arm_topology); } diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index 6d2be994ae58..50136828f5b5 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -9,15 +9,13 @@ #include <linux/sizes.h> -#include <asm-generic/vmlinux.lds.h> +#include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> #include <asm/memory.h> #include <asm/mpu.h> #include <asm/page.h> -#include "vmlinux.lds.h" - OUTPUT_ARCH(arm) ENTRY(stext) @@ -152,6 +150,10 @@ SECTIONS _end = .; STABS_DEBUG + DWARF_DEBUG + ARM_DETAILS + + ARM_ASSERTS } /* diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index 7f24bc08403e..5f4922e858d0 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -9,15 +9,13 @@ #else #include <linux/pgtable.h> -#include <asm-generic/vmlinux.lds.h> +#include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> #include <asm/memory.h> #include <asm/mpu.h> #include <asm/page.h> -#include "vmlinux.lds.h" - OUTPUT_ARCH(arm) ENTRY(stext) @@ -151,6 +149,10 @@ SECTIONS _end = .; STABS_DEBUG + DWARF_DEBUG + ARM_DETAILS + + ARM_ASSERTS } #ifdef CONFIG_STRICT_KERNEL_RWX diff --git a/arch/arm/lib/csumpartialcopy.S b/arch/arm/lib/csumpartialcopy.S index 184d97254a7a..1ca6aadd649c 100644 --- a/arch/arm/lib/csumpartialcopy.S +++ b/arch/arm/lib/csumpartialcopy.S @@ -9,8 +9,8 @@ .text -/* Function: __u32 csum_partial_copy_nocheck(const char *src, char *dst, int len, __u32 sum) - * Params : r0 = src, r1 = dst, r2 = len, r3 = checksum +/* Function: __u32 csum_partial_copy_nocheck(const char *src, char *dst, int len) + * Params : r0 = src, r1 = dst, r2 = len * Returns : r0 = new checksum */ diff --git a/arch/arm/lib/csumpartialcopygeneric.S b/arch/arm/lib/csumpartialcopygeneric.S index 0b706a39a677..0fd5c10e90a7 100644 --- a/arch/arm/lib/csumpartialcopygeneric.S +++ b/arch/arm/lib/csumpartialcopygeneric.S @@ -86,6 +86,7 @@ sum .req r3 FN_ENTRY save_regs + mov sum, #-1 cmp len, #8 @ Ensure that we have at least blo .Lless8 @ 8 bytes to copy. diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index 6bd3a93eaa3c..6928781e6bee 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -62,9 +62,9 @@ /* * unsigned int - * csum_partial_copy_from_user(const char *src, char *dst, int len, int sum, int *err_ptr) - * r0 = src, r1 = dst, r2 = len, r3 = sum, [sp] = *err_ptr - * Returns : r0 = checksum, [[sp, #0], #0] = 0 or -EFAULT + * csum_partial_copy_from_user(const char *src, char *dst, int len) + * r0 = src, r1 = dst, r2 = len + * Returns : r0 = checksum or 0 */ #define FN_ENTRY ENTRY(csum_partial_copy_from_user) @@ -73,25 +73,11 @@ #include "csumpartialcopygeneric.S" /* - * FIXME: minor buglet here - * We don't return the checksum for the data present in the buffer. To do - * so properly, we would have to add in whatever registers were loaded before - * the fault, which, with the current asm above is not predictable. + * We report fault by returning 0 csum - impossible in normal case, since + * we start with 0xffffffff for initial sum. */ .pushsection .text.fixup,"ax" .align 4 -9001: mov r4, #-EFAULT -#ifdef CONFIG_CPU_SW_DOMAIN_PAN - ldr r5, [sp, #9*4] @ *err_ptr -#else - ldr r5, [sp, #8*4] @ *err_ptr -#endif - str r4, [r5] - ldmia sp, {r1, r2} @ retrieve dst, len - add r2, r2, r1 - mov r0, #0 @ zero the buffer -9002: teq r2, r1 - strbne r0, [r1], #1 - bne 9002b +9001: mov r0, #0 load_regs .popsection diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 000c1b48e973..45f9d5ec2360 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -299,16 +299,14 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn) */ static void __init free_unused_memmap(void) { - unsigned long start, prev_end = 0; - struct memblock_region *reg; + unsigned long start, end, prev_end = 0; + int i; /* * This relies on each bank being in address order. * The banks are sorted previously in bootmem_init(). */ - for_each_memblock(memory, reg) { - start = memblock_region_memory_base_pfn(reg); - + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { #ifdef CONFIG_SPARSEMEM /* * Take care not to free memmap entries that don't exist @@ -336,8 +334,7 @@ static void __init free_unused_memmap(void) * memmap entries are valid from the bank end aligned to * MAX_ORDER_NR_PAGES. */ - prev_end = ALIGN(memblock_region_memory_end_pfn(reg), - MAX_ORDER_NR_PAGES); + prev_end = ALIGN(end, MAX_ORDER_NR_PAGES); } #ifdef CONFIG_SPARSEMEM @@ -347,61 +344,29 @@ static void __init free_unused_memmap(void) #endif } -#ifdef CONFIG_HIGHMEM -static inline void free_area_high(unsigned long pfn, unsigned long end) -{ - for (; pfn < end; pfn++) - free_highmem_page(pfn_to_page(pfn)); -} -#endif - static void __init free_highpages(void) { #ifdef CONFIG_HIGHMEM unsigned long max_low = max_low_pfn; - struct memblock_region *mem, *res; + phys_addr_t range_start, range_end; + u64 i; /* set highmem page free */ - for_each_memblock(memory, mem) { - unsigned long start = memblock_region_memory_base_pfn(mem); - unsigned long end = memblock_region_memory_end_pfn(mem); + for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, + &range_start, &range_end, NULL) { + unsigned long start = PHYS_PFN(range_start); + unsigned long end = PHYS_PFN(range_end); /* Ignore complete lowmem entries */ if (end <= max_low) continue; - if (memblock_is_nomap(mem)) - continue; - /* Truncate partial highmem entries */ if (start < max_low) start = max_low; - /* Find and exclude any reserved regions */ - for_each_memblock(reserved, res) { - unsigned long res_start, res_end; - - res_start = memblock_region_reserved_base_pfn(res); - res_end = memblock_region_reserved_end_pfn(res); - - if (res_end < start) - continue; - if (res_start < start) - res_start = start; - if (res_start > end) - res_start = end; - if (res_end > end) - res_end = end; - if (res_start != start) - free_area_high(start, res_start); - start = res_end; - if (start == end) - break; - } - - /* And now free anything which remains */ - if (start < end) - free_area_high(start, end); + for (; start < end; start++) + free_highmem_page(pfn_to_page(start)); } #endif } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c36f977b2ccb..698cc740c6b8 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1154,9 +1154,8 @@ phys_addr_t arm_lowmem_limit __initdata = 0; void __init adjust_lowmem_bounds(void) { - phys_addr_t memblock_limit = 0; - u64 vmalloc_limit; - struct memblock_region *reg; + phys_addr_t block_start, block_end, memblock_limit = 0; + u64 vmalloc_limit, i; phys_addr_t lowmem_limit = 0; /* @@ -1172,26 +1171,18 @@ void __init adjust_lowmem_bounds(void) * The first usable region must be PMD aligned. Mark its start * as MEMBLOCK_NOMAP if it isn't */ - for_each_memblock(memory, reg) { - if (!memblock_is_nomap(reg)) { - if (!IS_ALIGNED(reg->base, PMD_SIZE)) { - phys_addr_t len; + for_each_mem_range(i, &block_start, &block_end) { + if (!IS_ALIGNED(block_start, PMD_SIZE)) { + phys_addr_t len; - len = round_up(reg->base, PMD_SIZE) - reg->base; - memblock_mark_nomap(reg->base, len); - } - break; + len = round_up(block_start, PMD_SIZE) - block_start; + memblock_mark_nomap(block_start, len); } + break; } - for_each_memblock(memory, reg) { - phys_addr_t block_start = reg->base; - phys_addr_t block_end = reg->base + reg->size; - - if (memblock_is_nomap(reg)) - continue; - - if (reg->base < vmalloc_limit) { + for_each_mem_range(i, &block_start, &block_end) { + if (block_start < vmalloc_limit) { if (block_end > lowmem_limit) /* * Compare as u64 to ensure vmalloc_limit does @@ -1440,19 +1431,15 @@ static void __init kmap_init(void) static void __init map_lowmem(void) { - struct memblock_region *reg; phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE); phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + phys_addr_t start, end; + u64 i; /* Map all the lowmem memory banks. */ - for_each_memblock(memory, reg) { - phys_addr_t start = reg->base; - phys_addr_t end = start + reg->size; + for_each_mem_range(i, &start, &end) { struct map_desc map; - if (memblock_is_nomap(reg)) - continue; - if (end > arm_lowmem_limit) end = arm_lowmem_limit; if (start >= end) diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c index 699fa2e88725..88950e41a3a9 100644 --- a/arch/arm/mm/pmsa-v7.c +++ b/arch/arm/mm/pmsa-v7.c @@ -231,12 +231,12 @@ static int __init allocate_region(phys_addr_t base, phys_addr_t size, void __init pmsav7_adjust_lowmem_bounds(void) { phys_addr_t specified_mem_size = 0, total_mem_size = 0; - struct memblock_region *reg; - bool first = true; phys_addr_t mem_start; phys_addr_t mem_end; + phys_addr_t reg_start, reg_end; unsigned int mem_max_regions; - int num, i; + int num; + u64 i; /* Free-up PMSAv7_PROBE_REGION */ mpu_min_region_order = __mpu_min_region_order(); @@ -262,20 +262,19 @@ void __init pmsav7_adjust_lowmem_bounds(void) mem_max_regions -= num; #endif - for_each_memblock(memory, reg) { - if (first) { + for_each_mem_range(i, ®_start, ®_end) { + if (i == 0) { phys_addr_t phys_offset = PHYS_OFFSET; /* * Initially only use memory continuous from * PHYS_OFFSET */ - if (reg->base != phys_offset) + if (reg_start != phys_offset) panic("First memory bank must be contiguous from PHYS_OFFSET"); - mem_start = reg->base; - mem_end = reg->base + reg->size; - specified_mem_size = reg->size; - first = false; + mem_start = reg_start; + mem_end = reg_end; + specified_mem_size = mem_end - mem_start; } else { /* * memblock auto merges contiguous blocks, remove @@ -283,8 +282,8 @@ void __init pmsav7_adjust_lowmem_bounds(void) * blocks separately while iterating) */ pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", - &mem_end, ®->base); - memblock_remove(reg->base, 0 - reg->base); + &mem_end, ®_start); + memblock_remove(reg_start, 0 - reg_start); break; } } diff --git a/arch/arm/mm/pmsa-v8.c b/arch/arm/mm/pmsa-v8.c index 0d7d5fb59247..2de019f7503e 100644 --- a/arch/arm/mm/pmsa-v8.c +++ b/arch/arm/mm/pmsa-v8.c @@ -94,20 +94,19 @@ static __init bool is_region_fixed(int number) void __init pmsav8_adjust_lowmem_bounds(void) { phys_addr_t mem_end; - struct memblock_region *reg; - bool first = true; + phys_addr_t reg_start, reg_end; + u64 i; - for_each_memblock(memory, reg) { - if (first) { + for_each_mem_range(i, ®_start, ®_end) { + if (i == 0) { phys_addr_t phys_offset = PHYS_OFFSET; /* * Initially only use memory continuous from * PHYS_OFFSET */ - if (reg->base != phys_offset) + if (reg_start != phys_offset) panic("First memory bank must be contiguous from PHYS_OFFSET"); - mem_end = reg->base + reg->size; - first = false; + mem_end = reg_end; } else { /* * memblock auto merges contiguous blocks, remove @@ -115,8 +114,8 @@ void __init pmsav8_adjust_lowmem_bounds(void) * blocks separately while iterating) */ pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", - &mem_end, ®->base); - memblock_remove(reg->base, 0 - reg->base); + &mem_end, ®_start); + memblock_remove(reg_start, 0 - reg_start); break; } } diff --git a/arch/arm/probes/kprobes/core.c b/arch/arm/probes/kprobes/core.c index feefa2055eba..a9653117ca0d 100644 --- a/arch/arm/probes/kprobes/core.c +++ b/arch/arm/probes/kprobes/core.c @@ -413,87 +413,15 @@ void __naked __kprobes kretprobe_trampoline(void) /* Called from kretprobe_trampoline */ static __used __kprobes void *trampoline_handler(struct pt_regs *regs) { - struct kretprobe_instance *ri = NULL; - struct hlist_head *head, empty_rp; - struct hlist_node *tmp; - unsigned long flags, orig_ret_address = 0; - unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline; - kprobe_opcode_t *correct_ret_addr = NULL; - - INIT_HLIST_HEAD(&empty_rp); - kretprobe_hash_lock(current, &head, &flags); - - /* - * It is possible to have multiple instances associated with a given - * task either because multiple functions in the call path have - * a return probe installed on them, and/or more than one return - * probe was registered for a target function. - * - * We can handle this because: - * - instances are always inserted at the head of the list - * - when multiple return probes are registered for the same - * function, the first instance's ret_addr will point to the - * real return address, and all the rest will point to - * kretprobe_trampoline - */ - hlist_for_each_entry_safe(ri, tmp, head, hlist) { - if (ri->task != current) - /* another task is sharing our hash bucket */ - continue; - - orig_ret_address = (unsigned long)ri->ret_addr; - - if (orig_ret_address != trampoline_address) - /* - * This is the real return address. Any other - * instances associated with this task are for - * other calls deeper on the call stack - */ - break; - } - - kretprobe_assert(ri, orig_ret_address, trampoline_address); - - correct_ret_addr = ri->ret_addr; - hlist_for_each_entry_safe(ri, tmp, head, hlist) { - if (ri->task != current) - /* another task is sharing our hash bucket */ - continue; - - orig_ret_address = (unsigned long)ri->ret_addr; - if (ri->rp && ri->rp->handler) { - __this_cpu_write(current_kprobe, &ri->rp->kp); - get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; - ri->ret_addr = correct_ret_addr; - ri->rp->handler(ri, regs); - __this_cpu_write(current_kprobe, NULL); - } - - recycle_rp_inst(ri, &empty_rp); - - if (orig_ret_address != trampoline_address) - /* - * This is the real return address. Any other - * instances associated with this task are for - * other calls deeper on the call stack - */ - break; - } - - kretprobe_hash_unlock(current, &flags); - - hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { - hlist_del(&ri->hlist); - kfree(ri); - } - - return (void *)orig_ret_address; + return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, + (void *)regs->ARM_fp); } void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr; + ri->fp = (void *)regs->ARM_fp; /* Replace the return addr with trampoline addr. */ regs->ARM_lr = (unsigned long)&kretprobe_trampoline; diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index ea76562af1e9..60e901cd0de6 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -150,7 +150,7 @@ static int xen_starting_cpu(unsigned int cpu) pr_info("Xen: initializing cpu%d\n", cpu); vcpup = per_cpu_ptr(xen_vcpu_info, cpu); - info.mfn = virt_to_gfn(vcpup); + info.mfn = percpu_to_gfn(vcpup); info.offset = xen_offset_in_page(vcpup); err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu), diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 396797ffe2b1..d3ef975a0965 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -25,11 +25,12 @@ unsigned long xen_get_swiotlb_free_pages(unsigned int order) { - struct memblock_region *reg; + phys_addr_t base; gfp_t flags = __GFP_NOWARN|__GFP_KSWAPD_RECLAIM; + u64 i; - for_each_memblock(memory, reg) { - if (reg->base < (phys_addr_t)0xffffffff) { + for_each_mem_range(i, &base, NULL) { + if (base < (phys_addr_t)0xffffffff) { if (IS_ENABLED(CONFIG_ZONE_DMA32)) flags |= __GFP_DMA32; else |