From 4a3575fd436aa98957184afd745e4ada8f1542d8 Mon Sep 17 00:00:00 2001 From: "Huang, Ying" Date: Mon, 25 Feb 2008 15:18:37 +0800 Subject: x86: EFI_PAGE_SHIFT fix Make x86 EFI code works when EFI_PAGE_SHIFT != PAGE_SHIFT. The memrage_efi_to_native() provided in this patch can be used on other EFI platform such as IA64 too. This patch has been tested on Intel x86_64 platform with EFI 64/32 firmware. Signed-off-by: Huang Ying Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/linux/efi.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include') diff --git a/include/linux/efi.h b/include/linux/efi.h index 14813b595802..a5f359a7ad0e 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -394,4 +395,10 @@ struct efi_generic_dev_path { u16 length; } __attribute ((packed)); +static inline void memrange_efi_to_native(u64 *addr, u64 *npages) +{ + *npages = PFN_UP(*addr + (*npages< Date: Tue, 4 Mar 2008 22:05:27 -0800 Subject: x86: pageattr.c fix shadowed variable warning irqs_disabled() uses flags internally, use _flags to avoid shadowing code calling into this macro. Introduced between 2.6.25-rc3 and -rc4 Fixes the sparse warning: arch/x86/mm/pageattr.c:383:21: warning: symbol 'flags' shadows an earlier one arch/x86/mm/pageattr.c:369:16: originally declared here Signed-off-by: Harvey Harrison Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/linux/irqflags.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index 412e025bc5c7..e600c4e9b8c5 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -84,10 +84,10 @@ #define irqs_disabled() \ ({ \ - unsigned long flags; \ + unsigned long _flags; \ \ - raw_local_save_flags(flags); \ - raw_irqs_disabled_flags(flags); \ + raw_local_save_flags(_flags); \ + raw_irqs_disabled_flags(_flags); \ }) #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) -- cgit v1.2.3 From 7c53976404e2f906c60b69cc5793add87ee49c6a Mon Sep 17 00:00:00 2001 From: Alexander van Heukelum Date: Tue, 8 Apr 2008 12:54:30 +0200 Subject: x86: cleanup boot-heap usage The kernel decompressor wrapper uses memory located beyond the end of the image. This might lead to hard to debug problems, but even if it can be proven to be safe, it is at the very least unclean. I don't see any advantages either, unless you count it not being zeroed out as an advantage. This patch moves the boot-heap area to the bss segment. Signed-off-by: Alexander van Heukelum Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/boot/compressed/head_32.S | 15 +++++++++------ arch/x86/boot/compressed/head_64.S | 22 +++++++++++++--------- arch/x86/boot/compressed/misc.c | 8 +------- include/asm-x86/boot.h | 8 ++++++++ 4 files changed, 31 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index 036e635f18a3..ba7736cf2ec7 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -130,7 +130,7 @@ relocated: /* * Setup the stack for the decompressor */ - leal stack_end(%ebx), %esp + leal boot_stack_end(%ebx), %esp /* * Do the decompression, and jump to the new kernel.. @@ -142,8 +142,8 @@ relocated: pushl %eax # input_len leal input_data(%ebx), %eax pushl %eax # input_data - leal _end(%ebx), %eax - pushl %eax # end of the image as third argument + leal boot_heap(%ebx), %eax + pushl %eax # heap area as third argument pushl %esi # real mode pointer as second arg call decompress_kernel addl $20, %esp @@ -181,7 +181,10 @@ relocated: jmp *%ebp .bss +/* Stack and heap for uncompression */ .balign 4 -stack: - .fill 4096, 1, 0 -stack_end: +boot_heap: + .fill BOOT_HEAP_SIZE, 1, 0 +boot_stack: + .fill BOOT_STACK_SIZE, 1, 0 +boot_stack_end: diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index e8657b98c902..7a212a62db36 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -62,7 +63,7 @@ startup_32: subl $1b, %ebp /* setup a stack and make sure cpu supports long mode. */ - movl $user_stack_end, %eax + movl $boot_stack_end, %eax addl %ebp, %eax movl %eax, %esp @@ -274,7 +275,7 @@ relocated: stosb /* Setup the stack */ - leaq user_stack_end(%rip), %rsp + leaq boot_stack_end(%rip), %rsp /* zero EFLAGS after setting rsp */ pushq $0 @@ -285,7 +286,7 @@ relocated: */ pushq %rsi # Save the real mode argument movq %rsi, %rdi # real mode address - leaq _heap(%rip), %rsi # _heap + leaq boot_heap(%rip), %rsi # malloc area for uncompression leaq input_data(%rip), %rdx # input_data movl input_len(%rip), %eax movq %rax, %rcx # input_len @@ -310,9 +311,12 @@ gdt: .quad 0x0080890000000000 /* TS descriptor */ .quad 0x0000000000000000 /* TS continued */ gdt_end: - .bss -/* Stack for uncompression */ - .balign 4 -user_stack: - .fill 4096,4,0 -user_stack_end: + +.bss +/* Stack and heap for uncompression */ +.balign 4 +boot_heap: + .fill BOOT_HEAP_SIZE, 1, 0 +boot_stack: + .fill BOOT_STACK_SIZE, 1, 0 +boot_stack_end: diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index dad4e699f5a3..90456cee47c3 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -217,12 +217,6 @@ static void putstr(const char *); static memptr free_mem_ptr; static memptr free_mem_end_ptr; -#ifdef CONFIG_X86_64 -#define HEAP_SIZE 0x7000 -#else -#define HEAP_SIZE 0x4000 -#endif - static char *vidmem; static int vidport; static int lines, cols; @@ -449,7 +443,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap, window = output; /* Output buffer (Normally at 1M) */ free_mem_ptr = heap; /* Heap */ - free_mem_end_ptr = heap + HEAP_SIZE; + free_mem_end_ptr = heap + BOOT_HEAP_SIZE; inbuf = input_data; /* Input buffer */ insize = input_len; inptr = 0; diff --git a/include/asm-x86/boot.h b/include/asm-x86/boot.h index ed8affbf96cb..2faed7ecb092 100644 --- a/include/asm-x86/boot.h +++ b/include/asm-x86/boot.h @@ -17,4 +17,12 @@ + (CONFIG_PHYSICAL_ALIGN - 1)) \ & ~(CONFIG_PHYSICAL_ALIGN - 1)) +#ifdef CONFIG_X86_64 +#define BOOT_HEAP_SIZE 0x7000 +#define BOOT_STACK_SIZE 0x4000 +#else +#define BOOT_HEAP_SIZE 0x4000 +#define BOOT_STACK_SIZE 0x1000 +#endif + #endif /* _ASM_BOOT_H */ -- cgit v1.2.3 From 8fb402bccf203ecca8f9e0202b8fd3c937dece6f Mon Sep 17 00:00:00 2001 From: Erik Bosman Date: Fri, 11 Apr 2008 18:54:17 +0200 Subject: generic, x86: add prctl commands PR_GET_TSC and PR_SET_TSC This patch adds prctl commands that make it possible to deny the execution of timestamp counters in userspace. If this is not implemented on a specific architecture, prctl will return -EINVAL. ned-off-by: Erik Bosman Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/linux/prctl.h | 6 ++++++ kernel/sys.c | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/prctl.h b/include/linux/prctl.h index 3800639775ae..5c80b1939636 100644 --- a/include/linux/prctl.h +++ b/include/linux/prctl.h @@ -67,4 +67,10 @@ #define PR_CAPBSET_READ 23 #define PR_CAPBSET_DROP 24 +/* Get/set the process' ability to use the timestamp counter instruction */ +#define PR_GET_TSC 25 +#define PR_SET_TSC 26 +# define PR_TSC_ENABLE 1 /* allow the use of the timestamp counter */ +# define PR_TSC_SIGSEGV 2 /* throw a SIGSEGV instead of reading the TSC */ + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/sys.c b/kernel/sys.c index a626116af5db..6a0cc71ee88d 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -67,6 +67,12 @@ #ifndef SET_ENDIAN # define SET_ENDIAN(a,b) (-EINVAL) #endif +#ifndef GET_TSC_CTL +# define GET_TSC_CTL(a) (-EINVAL) +#endif +#ifndef SET_TSC_CTL +# define SET_TSC_CTL(a) (-EINVAL) +#endif /* * this is where the system-wide overflow UID and GID are defined, for @@ -1737,7 +1743,12 @@ asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3, #else return -EINVAL; #endif - + case PR_GET_TSC: + error = GET_TSC_CTL(arg2); + break; + case PR_SET_TSC: + error = SET_TSC_CTL(arg2); + break; default: error = -EINVAL; break; -- cgit v1.2.3 From 529e25f646e08901a6dad5768f681efffd77225e Mon Sep 17 00:00:00 2001 From: Erik Bosman Date: Mon, 14 Apr 2008 00:24:18 +0200 Subject: x86: implement prctl PR_GET_TSC and PR_SET_TSC This patch implements the PR_GET_TSC and PR_SET_TSC prctl() commands on the x86 platform (both 32 and 64 bit.) These commands control the ability to read the timestamp counter from userspace (the RDTSC instruction.) While the RDTSC instuction is a useful profiling tool, it is also the source of some non-determinism in ring-3. For deterministic replay applications it is useful to be able to trap and emulate (and record the outcome of) this instruction. This patch uses code earlier used to disable the timestamp counter for the SECCOMP framework. A side-effect of this patch is that the SECCOMP environment will now also disable the timestamp counter on x86_64 due to the addition of the TIF_NOTSC define on this platform. The code which enables/disables the RDTSC instruction during context switches is in the __switch_to_xtra function, which already handles other unusual conditions, so normal performance should not have to suffer from this change. Signed-off-by: Erik Bosman Acked-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/process_32.c | 43 ++++++++++++++++++++++--- arch/x86/kernel/process_64.c | 68 ++++++++++++++++++++++++++++++++++++++++ include/asm-x86/processor.h | 7 +++++ include/asm-x86/thread_info_64.h | 4 ++- include/asm-x86/tsc.h | 1 + 5 files changed, 118 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 91e147b486dd..a3790a3f8a83 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -523,11 +524,11 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) } EXPORT_SYMBOL_GPL(start_thread); -#ifdef CONFIG_SECCOMP static void hard_disable_TSC(void) { write_cr4(read_cr4() | X86_CR4_TSD); } + void disable_TSC(void) { preempt_disable(); @@ -539,11 +540,47 @@ void disable_TSC(void) hard_disable_TSC(); preempt_enable(); } + static void hard_enable_TSC(void) { write_cr4(read_cr4() & ~X86_CR4_TSD); } -#endif /* CONFIG_SECCOMP */ + +void enable_TSC(void) +{ + preempt_disable(); + if (test_and_clear_thread_flag(TIF_NOTSC)) + /* + * Must flip the CPU state synchronously with + * TIF_NOTSC in the current running context. + */ + hard_enable_TSC(); + preempt_enable(); +} + +int get_tsc_mode(unsigned long adr) +{ + unsigned int val; + + if (test_thread_flag(TIF_NOTSC)) + val = PR_TSC_SIGSEGV; + else + val = PR_TSC_ENABLE; + + return put_user(val, (unsigned int __user *)adr); +} + +int set_tsc_mode(unsigned int val) +{ + if (val == PR_TSC_SIGSEGV) + disable_TSC(); + else if (val == PR_TSC_ENABLE) + enable_TSC(); + else + return -EINVAL; + + return 0; +} static noinline void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, @@ -577,7 +614,6 @@ __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, set_debugreg(next->debugreg7, 7); } -#ifdef CONFIG_SECCOMP if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^ test_tsk_thread_flag(next_p, TIF_NOTSC)) { /* prev and next are different */ @@ -586,7 +622,6 @@ __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, else hard_enable_TSC(); } -#endif #ifdef X86_BTS if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index e75ccc8a2b87..4c13b1406c70 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -535,6 +536,64 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) } EXPORT_SYMBOL_GPL(start_thread); +static void hard_disable_TSC(void) +{ + write_cr4(read_cr4() | X86_CR4_TSD); +} + +void disable_TSC(void) +{ + preempt_disable(); + if (!test_and_set_thread_flag(TIF_NOTSC)) + /* + * Must flip the CPU state synchronously with + * TIF_NOTSC in the current running context. + */ + hard_disable_TSC(); + preempt_enable(); +} + +static void hard_enable_TSC(void) +{ + write_cr4(read_cr4() & ~X86_CR4_TSD); +} + +void enable_TSC(void) +{ + preempt_disable(); + if (test_and_clear_thread_flag(TIF_NOTSC)) + /* + * Must flip the CPU state synchronously with + * TIF_NOTSC in the current running context. + */ + hard_enable_TSC(); + preempt_enable(); +} + +int get_tsc_mode(unsigned long adr) +{ + unsigned int val; + + if (test_thread_flag(TIF_NOTSC)) + val = PR_TSC_SIGSEGV; + else + val = PR_TSC_ENABLE; + + return put_user(val, (unsigned int __user *)adr); +} + +int set_tsc_mode(unsigned int val) +{ + if (val == PR_TSC_SIGSEGV) + disable_TSC(); + else if (val == PR_TSC_ENABLE) + enable_TSC(); + else + return -EINVAL; + + return 0; +} + /* * This special macro can be used to load a debugging register */ @@ -572,6 +631,15 @@ static inline void __switch_to_xtra(struct task_struct *prev_p, loaddebug(next, 7); } + if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^ + test_tsk_thread_flag(next_p, TIF_NOTSC)) { + /* prev and next are different */ + if (test_tsk_thread_flag(next_p, TIF_NOTSC)) + hard_disable_TSC(); + else + hard_enable_TSC(); + } + if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { /* * Copy the relevant range of the IO bitmap. diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index 6e26c7c717a2..eaf4548a23d2 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -918,4 +918,11 @@ extern void start_thread(struct pt_regs *regs, unsigned long new_ip, #define KSTK_EIP(task) (task_pt_regs(task)->ip) +/* Get/set a process' ability to use the timestamp counter instruction */ +#define GET_TSC_CTL(adr) get_tsc_mode((adr)) +#define SET_TSC_CTL(val) set_tsc_mode((val)) + +extern int get_tsc_mode(unsigned long adr); +extern int set_tsc_mode(unsigned int val); + #endif diff --git a/include/asm-x86/thread_info_64.h b/include/asm-x86/thread_info_64.h index 1e5c6f6152cd..b17f5f6c2c59 100644 --- a/include/asm-x86/thread_info_64.h +++ b/include/asm-x86/thread_info_64.h @@ -126,6 +126,7 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ #define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */ #define TIF_BTS_TRACE_TS 27 /* record scheduling event timestamps */ +#define TIF_NOTSC 28 /* TSC is not accessible in userland */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) @@ -147,6 +148,7 @@ static inline struct thread_info *stack_thread_info(void) #define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) #define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR) #define _TIF_BTS_TRACE_TS (1 << TIF_BTS_TRACE_TS) +#define _TIF_NOTSC (1 << TIF_NOTSC) /* work to do on interrupt/exception return */ #define _TIF_WORK_MASK \ @@ -160,7 +162,7 @@ static inline struct thread_info *stack_thread_info(void) /* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW \ - (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_BTS_TRACE_TS) + (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_BTS_TRACE_TS|_TIF_NOTSC) #define _TIF_WORK_CTXSW_PREV _TIF_WORK_CTXSW #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) diff --git a/include/asm-x86/tsc.h b/include/asm-x86/tsc.h index d2d8eb5b55f5..0434bd8349a7 100644 --- a/include/asm-x86/tsc.h +++ b/include/asm-x86/tsc.h @@ -18,6 +18,7 @@ extern unsigned int cpu_khz; extern unsigned int tsc_khz; extern void disable_TSC(void); +extern void enable_TSC(void); static inline cycles_t get_cycles(void) { -- cgit v1.2.3 From fa5c4639419668cbb18ca3d20c1253559a3b43ae Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 16 Apr 2008 02:29:42 +0200 Subject: x86: rename find_max_pfn() to propagate_e820_map() this function doesnt just 'find' the max_pfn - it also has other side-effects such as registering sparse memory maps. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/e820_32.c | 4 ++-- arch/x86/kernel/setup_32.c | 4 ++-- arch/x86/mm/discontig_32.c | 6 +++--- include/asm-x86/e820_32.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/e820_32.c b/arch/x86/kernel/e820_32.c index 0240cd778365..ed733e7cf4e6 100644 --- a/arch/x86/kernel/e820_32.c +++ b/arch/x86/kernel/e820_32.c @@ -475,7 +475,7 @@ int __init copy_e820_map(struct e820entry *biosmap, int nr_map) /* * Find the highest page frame number we have available */ -void __init find_max_pfn(void) +void __init propagate_e820_map(void) { int i; @@ -704,7 +704,7 @@ static int __init parse_memmap(char *arg) * size before original memory map is * reset. */ - find_max_pfn(); + propagate_e820_map(); saved_max_pfn = max_pfn; #endif e820.nr_map = 0; diff --git a/arch/x86/kernel/setup_32.c b/arch/x86/kernel/setup_32.c index 5b0bffb7fcc9..1c4799e68718 100644 --- a/arch/x86/kernel/setup_32.c +++ b/arch/x86/kernel/setup_32.c @@ -812,10 +812,10 @@ void __init setup_arch(char **cmdline_p) efi_init(); /* update e820 for memory not covered by WB MTRRs */ - find_max_pfn(); + propagate_e820_map(); mtrr_bp_init(); if (mtrr_trim_uncached_memory(max_pfn)) - find_max_pfn(); + propagate_e820_map(); max_low_pfn = setup_memory(); diff --git a/arch/x86/mm/discontig_32.c b/arch/x86/mm/discontig_32.c index eba0bbede7a6..18378850e25a 100644 --- a/arch/x86/mm/discontig_32.c +++ b/arch/x86/mm/discontig_32.c @@ -120,7 +120,7 @@ int __init get_memcfg_numa_flat(void) printk("NUMA - single node, flat memory mode\n"); /* Run the memory configuration and find the top of memory. */ - find_max_pfn(); + propagate_e820_map(); node_start_pfn[0] = 0; node_end_pfn[0] = max_pfn; memory_present(0, 0, max_pfn); @@ -134,7 +134,7 @@ int __init get_memcfg_numa_flat(void) /* * Find the highest page frame number we have available for the node */ -static void __init find_max_pfn_node(int nid) +static void __init propagate_e820_map_node(int nid) { if (node_end_pfn[nid] > max_pfn) node_end_pfn[nid] = max_pfn; @@ -379,7 +379,7 @@ unsigned long __init setup_memory(void) printk("High memory starts at vaddr %08lx\n", (ulong) pfn_to_kaddr(highstart_pfn)); for_each_online_node(nid) - find_max_pfn_node(nid); + propagate_e820_map_node(nid); memset(NODE_DATA(0), 0, sizeof(struct pglist_data)); NODE_DATA(0)->bdata = &node0_bdata; diff --git a/include/asm-x86/e820_32.h b/include/asm-x86/e820_32.h index 43b1a8bd4b34..a9f7c6ec32bf 100644 --- a/include/asm-x86/e820_32.h +++ b/include/asm-x86/e820_32.h @@ -24,7 +24,7 @@ extern void update_e820(void); extern int e820_all_mapped(unsigned long start, unsigned long end, unsigned type); extern int e820_any_mapped(u64 start, u64 end, unsigned type); -extern void find_max_pfn(void); +extern void propagate_e820_map(void); extern void register_bootmem_low_pages(unsigned long max_low_pfn); extern void add_memory_region(unsigned long long start, unsigned long long size, int type); -- cgit v1.2.3 From 61c4628b538608c1a85211ed8438136adfeb9a95 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Mon, 10 Mar 2008 15:28:04 -0700 Subject: x86, fpu: split FPU state from task struct - v5 Split the FPU save area from the task struct. This allows easy migration of FPU context, and it's generally cleaner. It also allows the following two optimizations: 1) only allocate when the application actually uses FPU, so in the first lazy FPU trap. This could save memory for non-fpu using apps. Next patch does this lazy allocation. 2) allocate the right size for the actual cpu rather than 512 bytes always. Patches enabling xsave/xrstor support (coming shortly) will take advantage of this. Signed-off-by: Suresh Siddha Signed-off-by: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/i387.c | 80 +++++++++++++++++++++++----------------- arch/x86/kernel/process.c | 35 ++++++++++++++++++ arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 2 +- arch/x86/kernel/traps_32.c | 6 +-- arch/x86/kernel/traps_64.c | 6 ++- arch/x86/math-emu/fpu_entry.c | 4 +- arch/x86/math-emu/fpu_system.h | 26 ++++++------- arch/x86/math-emu/reg_ld_str.c | 4 +- include/asm-x86/i387.h | 35 +++++++++--------- include/asm-x86/processor.h | 7 ++-- include/asm-x86/thread_info.h | 8 ++++ include/asm-x86/thread_info_32.h | 2 - include/asm-x86/thread_info_64.h | 2 - kernel/fork.c | 31 +++++++++++++--- 16 files changed, 161 insertions(+), 90 deletions(-) create mode 100644 arch/x86/kernel/process.c (limited to 'include') diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index c3920ea8ac56..7a2a2e93e84b 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_X86_64) += pci-nommu_64.o bugs_64.o obj-y += tsc_$(BITS).o io_delay.o rtc.o obj-$(CONFIG_X86_TRAMPOLINE) += trampoline.o +obj-y += process.o obj-y += i387.o obj-y += ptrace.o obj-y += ds.o diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index 8f8102d967b3..baf632b221d4 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -35,17 +36,18 @@ #endif static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu; +unsigned int xstate_size; +static struct i387_fxsave_struct fx_scratch __cpuinitdata; -void mxcsr_feature_mask_init(void) +void __cpuinit mxcsr_feature_mask_init(void) { unsigned long mask = 0; clts(); if (cpu_has_fxsr) { - memset(¤t->thread.i387.fxsave, 0, - sizeof(struct i387_fxsave_struct)); - asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); - mask = current->thread.i387.fxsave.mxcsr_mask; + memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct)); + asm volatile("fxsave %0" : : "m" (fx_scratch)); + mask = fx_scratch.mxcsr_mask; if (mask == 0) mask = 0x0000ffbf; } @@ -53,6 +55,17 @@ void mxcsr_feature_mask_init(void) stts(); } +void __init init_thread_xstate(void) +{ + if (cpu_has_fxsr) + xstate_size = sizeof(struct i387_fxsave_struct); +#ifdef CONFIG_X86_32 + else + xstate_size = sizeof(struct i387_fsave_struct); +#endif + init_task.thread.xstate = alloc_bootmem(xstate_size); +} + #ifdef CONFIG_X86_64 /* * Called at bootup to set up the initial FPU state that is later cloned @@ -61,10 +74,6 @@ void mxcsr_feature_mask_init(void) void __cpuinit fpu_init(void) { unsigned long oldcr0 = read_cr0(); - extern void __bad_fxsave_alignment(void); - - if (offsetof(struct task_struct, thread.i387.fxsave) & 15) - __bad_fxsave_alignment(); set_in_cr4(X86_CR4_OSFXSR); set_in_cr4(X86_CR4_OSXMMEXCPT); @@ -93,18 +102,19 @@ void init_fpu(struct task_struct *tsk) } if (cpu_has_fxsr) { - memset(&tsk->thread.i387.fxsave, 0, - sizeof(struct i387_fxsave_struct)); - tsk->thread.i387.fxsave.cwd = 0x37f; + struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; + + memset(fx, 0, xstate_size); + fx->cwd = 0x37f; if (cpu_has_xmm) - tsk->thread.i387.fxsave.mxcsr = MXCSR_DEFAULT; + fx->mxcsr = MXCSR_DEFAULT; } else { - memset(&tsk->thread.i387.fsave, 0, - sizeof(struct i387_fsave_struct)); - tsk->thread.i387.fsave.cwd = 0xffff037fu; - tsk->thread.i387.fsave.swd = 0xffff0000u; - tsk->thread.i387.fsave.twd = 0xffffffffu; - tsk->thread.i387.fsave.fos = 0xffff0000u; + struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave; + memset(fp, 0, xstate_size); + fp->cwd = 0xffff037fu; + fp->swd = 0xffff0000u; + fp->twd = 0xffffffffu; + fp->fos = 0xffff0000u; } /* * Only the device not available exception or ptrace can call init_fpu. @@ -132,7 +142,7 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset, init_fpu(target); return user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &target->thread.i387.fxsave, 0, -1); + &target->thread.xstate->fxsave, 0, -1); } int xfpregs_set(struct task_struct *target, const struct user_regset *regset, @@ -148,12 +158,12 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, set_stopped_child_used_math(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - &target->thread.i387.fxsave, 0, -1); + &target->thread.xstate->fxsave, 0, -1); /* * mxcsr reserved bits must be masked to zero for security reasons. */ - target->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; + target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; return ret; } @@ -233,7 +243,7 @@ static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave) static void convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) { - struct i387_fxsave_struct *fxsave = &tsk->thread.i387.fxsave; + struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave; struct _fpreg *to = (struct _fpreg *) &env->st_space[0]; struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0]; int i; @@ -273,7 +283,7 @@ static void convert_to_fxsr(struct task_struct *tsk, const struct user_i387_ia32_struct *env) { - struct i387_fxsave_struct *fxsave = &tsk->thread.i387.fxsave; + struct i387_fxsave_struct *fxsave = &tsk->thread.xstate->fxsave; struct _fpreg *from = (struct _fpreg *) &env->st_space[0]; struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0]; int i; @@ -310,7 +320,8 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset, if (!cpu_has_fxsr) { return user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &target->thread.i387.fsave, 0, -1); + &target->thread.xstate->fsave, 0, + -1); } if (kbuf && pos == 0 && count == sizeof(env)) { @@ -338,7 +349,7 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, if (!cpu_has_fxsr) { return user_regset_copyin(&pos, &count, &kbuf, &ubuf, - &target->thread.i387.fsave, 0, -1); + &target->thread.xstate->fsave, 0, -1); } if (pos > 0 || count < sizeof(env)) @@ -358,11 +369,11 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf) { struct task_struct *tsk = current; + struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave; unlazy_fpu(tsk); - tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd; - if (__copy_to_user(buf, &tsk->thread.i387.fsave, - sizeof(struct i387_fsave_struct))) + fp->status = fp->swd; + if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct))) return -1; return 1; } @@ -370,6 +381,7 @@ static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf) static int save_i387_fxsave(struct _fpstate_ia32 __user *buf) { struct task_struct *tsk = current; + struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; struct user_i387_ia32_struct env; int err = 0; @@ -379,12 +391,12 @@ static int save_i387_fxsave(struct _fpstate_ia32 __user *buf) if (__copy_to_user(buf, &env, sizeof(env))) return -1; - err |= __put_user(tsk->thread.i387.fxsave.swd, &buf->status); + err |= __put_user(fx->swd, &buf->status); err |= __put_user(X86_FXSR_MAGIC, &buf->magic); if (err) return -1; - if (__copy_to_user(&buf->_fxsr_env[0], &tsk->thread.i387.fxsave, + if (__copy_to_user(&buf->_fxsr_env[0], fx, sizeof(struct i387_fxsave_struct))) return -1; return 1; @@ -417,7 +429,7 @@ static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf) struct task_struct *tsk = current; clear_fpu(tsk); - return __copy_from_user(&tsk->thread.i387.fsave, buf, + return __copy_from_user(&tsk->thread.xstate->fsave, buf, sizeof(struct i387_fsave_struct)); } @@ -428,10 +440,10 @@ static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf) int err; clear_fpu(tsk); - err = __copy_from_user(&tsk->thread.i387.fxsave, &buf->_fxsr_env[0], + err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0], sizeof(struct i387_fxsave_struct)); /* mxcsr reserved bits must be masked to zero for security reasons */ - tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; + tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; if (err || __copy_from_user(&env, buf, sizeof(env))) return 1; convert_to_fxsr(tsk, &env); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c new file mode 100644 index 000000000000..ead24efbcba0 --- /dev/null +++ b/arch/x86/kernel/process.c @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#include +#include + +static struct kmem_cache *task_xstate_cachep; + +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) +{ + *dst = *src; + dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); + if (!dst->thread.xstate) + return -ENOMEM; + WARN_ON((unsigned long)dst->thread.xstate & 15); + memcpy(dst->thread.xstate, src->thread.xstate, xstate_size); + return 0; +} + +void free_thread_info(struct thread_info *ti) +{ + kmem_cache_free(task_xstate_cachep, ti->task->thread.xstate); + ti->task->thread.xstate = NULL; + + free_pages((unsigned long)(ti), get_order(THREAD_SIZE)); +} + +void arch_task_cache_init(void) +{ + task_xstate_cachep = + kmem_cache_create("task_xstate", xstate_size, + __alignof__(union thread_xstate), + SLAB_PANIC, NULL); +} diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index a3790a3f8a83..3890a5dd25f9 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -703,7 +703,7 @@ struct task_struct * __switch_to(struct task_struct *prev_p, struct task_struct /* we're going to use this soon, after a few expensive things */ if (next_p->fpu_counter > 5) - prefetch(&next->i387.fxsave); + prefetch(next->xstate); /* * Reload esp0. diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 4c13b1406c70..b795e831afd6 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -682,7 +682,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) /* we're going to use this soon, after a few expensive things */ if (next_p->fpu_counter>5) - prefetch(&next->i387.fxsave); + prefetch(next->xstate); /* * Reload esp0, LDT and the page table pointer: diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c index dc4273010f2a..8d136a73ce8e 100644 --- a/arch/x86/kernel/traps_32.c +++ b/arch/x86/kernel/traps_32.c @@ -1208,11 +1208,6 @@ void __init trap_init(void) #endif set_trap_gate(19, &simd_coprocessor_error); - /* - * Verify that the FXSAVE/FXRSTOR data will be 16-byte aligned. - * Generate a build-time error if the alignment is wrong. - */ - BUILD_BUG_ON(offsetof(struct task_struct, thread.i387.fxsave) & 15); if (cpu_has_fxsr) { printk(KERN_INFO "Enabling fast FPU save and restore... "); set_in_cr4(X86_CR4_OSFXSR); @@ -1233,6 +1228,7 @@ void __init trap_init(void) set_bit(SYSCALL_VECTOR, used_vectors); + init_thread_xstate(); /* * Should be a barrier for any external CPU state: */ diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c index 6d883b13ef4f..dc0cb497eec3 100644 --- a/arch/x86/kernel/traps_64.c +++ b/arch/x86/kernel/traps_64.c @@ -1128,7 +1128,7 @@ asmlinkage void math_state_restore(void) if (!used_math()) init_fpu(me); - restore_fpu_checking(&me->thread.i387.fxsave); + restore_fpu_checking(&me->thread.xstate->fxsave); task_thread_info(me)->status |= TS_USEDFPU; me->fpu_counter++; } @@ -1163,6 +1163,10 @@ void __init trap_init(void) set_system_gate(IA32_SYSCALL_VECTOR, ia32_syscall); #endif + /* + * initialize the per thread extended state: + */ + init_thread_xstate(); /* * Should be a barrier for any external CPU state. */ diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c index 4bab3b145392..6e38d877ea77 100644 --- a/arch/x86/math-emu/fpu_entry.c +++ b/arch/x86/math-emu/fpu_entry.c @@ -678,7 +678,7 @@ int fpregs_soft_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct i387_soft_struct *s387 = &target->thread.i387.soft; + struct i387_soft_struct *s387 = &target->thread.xstate->soft; void *space = s387->st_space; int ret; int offset, other, i, tags, regnr, tag, newtop; @@ -730,7 +730,7 @@ int fpregs_soft_get(struct task_struct *target, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { - struct i387_soft_struct *s387 = &target->thread.i387.soft; + struct i387_soft_struct *s387 = &target->thread.xstate->soft; const void *space = s387->st_space; int ret; int offset = (S387->ftop & 7) * 10, other = 80 - offset; diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h index a3ae28c49ddd..13488fa153e0 100644 --- a/arch/x86/math-emu/fpu_system.h +++ b/arch/x86/math-emu/fpu_system.h @@ -35,8 +35,8 @@ #define SEG_EXPAND_DOWN(s) (((s).b & ((1 << 11) | (1 << 10))) \ == (1 << 10)) -#define I387 (current->thread.i387) -#define FPU_info (I387.soft.info) +#define I387 (current->thread.xstate) +#define FPU_info (I387->soft.info) #define FPU_CS (*(unsigned short *) &(FPU_info->___cs)) #define FPU_SS (*(unsigned short *) &(FPU_info->___ss)) @@ -46,25 +46,25 @@ #define FPU_EIP (FPU_info->___eip) #define FPU_ORIG_EIP (FPU_info->___orig_eip) -#define FPU_lookahead (I387.soft.lookahead) +#define FPU_lookahead (I387->soft.lookahead) /* nz if ip_offset and cs_selector are not to be set for the current instruction. */ -#define no_ip_update (*(u_char *)&(I387.soft.no_update)) -#define FPU_rm (*(u_char *)&(I387.soft.rm)) +#define no_ip_update (*(u_char *)&(I387->soft.no_update)) +#define FPU_rm (*(u_char *)&(I387->soft.rm)) /* Number of bytes of data which can be legally accessed by the current instruction. This only needs to hold a number <= 108, so a byte will do. */ -#define access_limit (*(u_char *)&(I387.soft.alimit)) +#define access_limit (*(u_char *)&(I387->soft.alimit)) -#define partial_status (I387.soft.swd) -#define control_word (I387.soft.cwd) -#define fpu_tag_word (I387.soft.twd) -#define registers (I387.soft.st_space) -#define top (I387.soft.ftop) +#define partial_status (I387->soft.swd) +#define control_word (I387->soft.cwd) +#define fpu_tag_word (I387->soft.twd) +#define registers (I387->soft.st_space) +#define top (I387->soft.ftop) -#define instruction_address (*(struct address *)&I387.soft.fip) -#define operand_address (*(struct address *)&I387.soft.foo) +#define instruction_address (*(struct address *)&I387->soft.fip) +#define operand_address (*(struct address *)&I387->soft.foo) #define FPU_access_ok(x,y,z) if ( !access_ok(x,y,z) ) \ math_abort(FPU_info,SIGSEGV) diff --git a/arch/x86/math-emu/reg_ld_str.c b/arch/x86/math-emu/reg_ld_str.c index 02af772a24db..d597fe7423c9 100644 --- a/arch/x86/math-emu/reg_ld_str.c +++ b/arch/x86/math-emu/reg_ld_str.c @@ -1180,8 +1180,8 @@ u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d) control_word |= 0xffff0040; partial_status = status_word() | 0xffff0000; fpu_tag_word |= 0xffff0000; - I387.soft.fcs &= ~0xf8000000; - I387.soft.fos |= 0xffff0000; + I387->soft.fcs &= ~0xf8000000; + I387->soft.fos |= 0xffff0000; #endif /* PECULIAR_486 */ if (__copy_to_user(d, &control_word, 7 * 4)) FPU_abort; diff --git a/include/asm-x86/i387.h b/include/asm-x86/i387.h index 54522b814f1c..382a5fa9d492 100644 --- a/include/asm-x86/i387.h +++ b/include/asm-x86/i387.h @@ -23,6 +23,7 @@ extern void fpu_init(void); extern void mxcsr_feature_mask_init(void); extern void init_fpu(struct task_struct *child); extern asmlinkage void math_state_restore(void); +extern void init_thread_xstate(void); extern user_regset_active_fn fpregs_active, xfpregs_active; extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; @@ -117,24 +118,22 @@ static inline void __save_init_fpu(struct task_struct *tsk) /* Using "fxsaveq %0" would be the ideal choice, but is only supported starting with gas 2.16. */ __asm__ __volatile__("fxsaveq %0" - : "=m" (tsk->thread.i387.fxsave)); + : "=m" (tsk->thread.xstate->fxsave)); #elif 0 /* Using, as a workaround, the properly prefixed form below isn't accepted by any binutils version so far released, complaining that the same type of prefix is used twice if an extended register is needed for addressing (fix submitted to mainline 2005-11-21). */ __asm__ __volatile__("rex64/fxsave %0" - : "=m" (tsk->thread.i387.fxsave)); + : "=m" (tsk->thread.xstate->fxsave)); #else /* This, however, we can work around by forcing the compiler to select an addressing mode that doesn't require extended registers. */ - __asm__ __volatile__("rex64/fxsave %P2(%1)" - : "=m" (tsk->thread.i387.fxsave) - : "cdaSDb" (tsk), - "i" (offsetof(__typeof__(*tsk), - thread.i387.fxsave))); + __asm__ __volatile__("rex64/fxsave (%1)" + : "=m" (tsk->thread.xstate->fxsave) + : "cdaSDb" (&tsk->thread.xstate->fxsave)); #endif - clear_fpu_state(&tsk->thread.i387.fxsave); + clear_fpu_state(&tsk->thread.xstate->fxsave); task_thread_info(tsk)->status &= ~TS_USEDFPU; } @@ -148,7 +147,7 @@ static inline int save_i387(struct _fpstate __user *buf) int err = 0; BUILD_BUG_ON(sizeof(struct user_i387_struct) != - sizeof(tsk->thread.i387.fxsave)); + sizeof(tsk->thread.xstate->fxsave)); if ((unsigned long)buf % 16) printk("save_i387: bad fpstate %p\n", buf); @@ -164,7 +163,7 @@ static inline int save_i387(struct _fpstate __user *buf) task_thread_info(tsk)->status &= ~TS_USEDFPU; stts(); } else { - if (__copy_to_user(buf, &tsk->thread.i387.fxsave, + if (__copy_to_user(buf, &tsk->thread.xstate->fxsave, sizeof(struct i387_fxsave_struct))) return -1; } @@ -201,7 +200,7 @@ static inline void restore_fpu(struct task_struct *tsk) "nop ; frstor %1", "fxrstor %1", X86_FEATURE_FXSR, - "m" ((tsk)->thread.i387.fxsave)); + "m" (tsk->thread.xstate->fxsave)); } /* We need a safe address that is cheap to find and that is already @@ -225,8 +224,8 @@ static inline void __save_init_fpu(struct task_struct *tsk) "fxsave %[fx]\n" "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", X86_FEATURE_FXSR, - [fx] "m" (tsk->thread.i387.fxsave), - [fsw] "m" (tsk->thread.i387.fxsave.swd) : "memory"); + [fx] "m" (tsk->thread.xstate->fxsave), + [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is pending. Clear the x87 state here by setting it to fixed values. safe_address is a random variable that should be in L1 */ @@ -327,25 +326,25 @@ static inline void clear_fpu(struct task_struct *tsk) static inline unsigned short get_fpu_cwd(struct task_struct *tsk) { if (cpu_has_fxsr) { - return tsk->thread.i387.fxsave.cwd; + return tsk->thread.xstate->fxsave.cwd; } else { - return (unsigned short)tsk->thread.i387.fsave.cwd; + return (unsigned short) tsk->thread.xstate->fsave.cwd; } } static inline unsigned short get_fpu_swd(struct task_struct *tsk) { if (cpu_has_fxsr) { - return tsk->thread.i387.fxsave.swd; + return tsk->thread.xstate->fxsave.swd; } else { - return (unsigned short)tsk->thread.i387.fsave.swd; + return (unsigned short) tsk->thread.xstate->fsave.swd; } } static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) { if (cpu_has_xmm) { - return tsk->thread.i387.fxsave.mxcsr; + return tsk->thread.xstate->fxsave.mxcsr; } else { return MXCSR_DEFAULT; } diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index eaf4548a23d2..99d297885780 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -354,7 +354,7 @@ struct i387_soft_struct { u32 entry_eip; }; -union i387_union { +union thread_xstate { struct i387_fsave_struct fsave; struct i387_fxsave_struct fxsave; struct i387_soft_struct soft; @@ -365,6 +365,7 @@ DECLARE_PER_CPU(struct orig_ist, orig_ist); #endif extern void print_cpu_info(struct cpuinfo_x86 *); +extern unsigned int xstate_size; extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); extern unsigned short num_cache_leaves; @@ -397,8 +398,8 @@ struct thread_struct { unsigned long cr2; unsigned long trap_no; unsigned long error_code; - /* Floating point info: */ - union i387_union i387 __attribute__((aligned(16)));; + /* floating point and extended processor state */ + union thread_xstate *xstate; #ifdef CONFIG_X86_32 /* Virtual 86 mode info */ struct vm86_struct __user *vm86_info; diff --git a/include/asm-x86/thread_info.h b/include/asm-x86/thread_info.h index d5fd12f2abdb..407b88c170d3 100644 --- a/include/asm-x86/thread_info.h +++ b/include/asm-x86/thread_info.h @@ -1,5 +1,13 @@ +#ifndef _ASM_X86_THREAD_INFO_H #ifdef CONFIG_X86_32 # include "thread_info_32.h" #else # include "thread_info_64.h" #endif + +#ifndef __ASSEMBLY__ +extern void arch_task_cache_init(void); +extern void free_thread_info(struct thread_info *ti); +extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); +#endif +#endif /* _ASM_X86_THREAD_INFO_H */ diff --git a/include/asm-x86/thread_info_32.h b/include/asm-x86/thread_info_32.h index 4e053fa561a9..531859962096 100644 --- a/include/asm-x86/thread_info_32.h +++ b/include/asm-x86/thread_info_32.h @@ -102,8 +102,6 @@ static inline struct thread_info *current_thread_info(void) __get_free_pages(GFP_KERNEL, get_order(THREAD_SIZE))) #endif -#define free_thread_info(info) free_pages((unsigned long)(info), get_order(THREAD_SIZE)) - #else /* !__ASSEMBLY__ */ /* how to get the thread information struct from ASM */ diff --git a/include/asm-x86/thread_info_64.h b/include/asm-x86/thread_info_64.h index b17f5f6c2c59..ed664e874dec 100644 --- a/include/asm-x86/thread_info_64.h +++ b/include/asm-x86/thread_info_64.h @@ -85,8 +85,6 @@ static inline struct thread_info *stack_thread_info(void) #define alloc_thread_info(tsk) \ ((struct thread_info *)__get_free_pages(THREAD_FLAGS, THREAD_ORDER)) -#define free_thread_info(ti) free_pages((unsigned long) (ti), THREAD_ORDER) - #else /* !__ASSEMBLY__ */ /* how to get the thread information struct from ASM */ diff --git a/kernel/fork.c b/kernel/fork.c index 9c042f901570..44a18192c420 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -132,6 +132,10 @@ void __put_task_struct(struct task_struct *tsk) free_task(tsk); } +void __attribute__((weak)) arch_task_cache_init(void) +{ +} + void __init fork_init(unsigned long mempages) { #ifndef __HAVE_ARCH_TASK_STRUCT_ALLOCATOR @@ -144,6 +148,9 @@ void __init fork_init(unsigned long mempages) ARCH_MIN_TASKALIGN, SLAB_PANIC, NULL); #endif + /* do the arch specific task caches init */ + arch_task_cache_init(); + /* * The default maximum number of threads is set to a safe * value: the thread structures can take up at most half @@ -163,6 +170,13 @@ void __init fork_init(unsigned long mempages) init_task.signal->rlim[RLIMIT_NPROC]; } +int __attribute__((weak)) arch_dup_task_struct(struct task_struct *dst, + struct task_struct *src) +{ + *dst = *src; + return 0; +} + static struct task_struct *dup_task_struct(struct task_struct *orig) { struct task_struct *tsk; @@ -181,15 +195,15 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) return NULL; } - *tsk = *orig; + err = arch_dup_task_struct(tsk, orig); + if (err) + goto out; + tsk->stack = ti; err = prop_local_init_single(&tsk->dirties); - if (err) { - free_thread_info(ti); - free_task_struct(tsk); - return NULL; - } + if (err) + goto out; setup_thread_stack(tsk, orig); @@ -205,6 +219,11 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) #endif tsk->splice_pipe = NULL; return tsk; + +out: + free_thread_info(ti); + free_task_struct(tsk); + return NULL; } #ifdef CONFIG_MMU -- cgit v1.2.3 From aa283f49276e7d840a40fb01eee6de97eaa7e012 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Mon, 10 Mar 2008 15:28:05 -0700 Subject: x86, fpu: lazy allocation of FPU area - v5 Only allocate the FPU area when the application actually uses FPU, i.e., in the first lazy FPU trap. This could save memory for non-fpu using apps. for example: on my system after boot, there are around 300 processes, with only 17 using FPU. Signed-off-by: Suresh Siddha Cc: Arjan van de Ven Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/i387.c | 38 ++++++++++++++++++++++++++++++-------- arch/x86/kernel/process.c | 28 +++++++++++++++++++--------- arch/x86/kernel/process_32.c | 4 ++++ arch/x86/kernel/process_64.c | 4 ++++ arch/x86/kernel/traps_32.c | 17 +++++++++++++++-- arch/x86/kernel/traps_64.c | 19 ++++++++++++++++--- include/asm-x86/i387.h | 2 +- include/asm-x86/processor.h | 2 ++ 8 files changed, 91 insertions(+), 23 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index baf632b221d4..db6839b53195 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -63,7 +62,6 @@ void __init init_thread_xstate(void) else xstate_size = sizeof(struct i387_fsave_struct); #endif - init_task.thread.xstate = alloc_bootmem(xstate_size); } #ifdef CONFIG_X86_64 @@ -93,12 +91,22 @@ void __cpuinit fpu_init(void) * value at reset if we support XMM instructions and then * remeber the current task has used the FPU. */ -void init_fpu(struct task_struct *tsk) +int init_fpu(struct task_struct *tsk) { if (tsk_used_math(tsk)) { if (tsk == current) unlazy_fpu(tsk); - return; + return 0; + } + + /* + * Memory allocation at the first usage of the FPU and other state. + */ + if (!tsk->thread.xstate) { + tsk->thread.xstate = kmem_cache_alloc(task_xstate_cachep, + GFP_KERNEL); + if (!tsk->thread.xstate) + return -ENOMEM; } if (cpu_has_fxsr) { @@ -120,6 +128,7 @@ void init_fpu(struct task_struct *tsk) * Only the device not available exception or ptrace can call init_fpu. */ set_stopped_child_used_math(tsk); + return 0; } int fpregs_active(struct task_struct *target, const struct user_regset *regset) @@ -136,10 +145,14 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, void *kbuf, void __user *ubuf) { + int ret; + if (!cpu_has_fxsr) return -ENODEV; - init_fpu(target); + ret = init_fpu(target); + if (ret) + return ret; return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.xstate->fxsave, 0, -1); @@ -154,7 +167,10 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, if (!cpu_has_fxsr) return -ENODEV; - init_fpu(target); + ret = init_fpu(target); + if (ret) + return ret; + set_stopped_child_used_math(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, @@ -312,11 +328,14 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset, void *kbuf, void __user *ubuf) { struct user_i387_ia32_struct env; + int ret; if (!HAVE_HWFP) return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf); - init_fpu(target); + ret = init_fpu(target); + if (ret) + return ret; if (!cpu_has_fxsr) { return user_regset_copyout(&pos, &count, &kbuf, &ubuf, @@ -344,7 +363,10 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, if (!HAVE_HWFP) return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf); - init_fpu(target); + ret = init_fpu(target); + if (ret) + return ret; + set_stopped_child_used_math(target); if (!cpu_has_fxsr) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index ead24efbcba0..0e613e7e7b5e 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -5,24 +5,34 @@ #include #include -static struct kmem_cache *task_xstate_cachep; +struct kmem_cache *task_xstate_cachep; int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { *dst = *src; - dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); - if (!dst->thread.xstate) - return -ENOMEM; - WARN_ON((unsigned long)dst->thread.xstate & 15); - memcpy(dst->thread.xstate, src->thread.xstate, xstate_size); + if (src->thread.xstate) { + dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep, + GFP_KERNEL); + if (!dst->thread.xstate) + return -ENOMEM; + WARN_ON((unsigned long)dst->thread.xstate & 15); + memcpy(dst->thread.xstate, src->thread.xstate, xstate_size); + } return 0; } -void free_thread_info(struct thread_info *ti) +void free_thread_xstate(struct task_struct *tsk) { - kmem_cache_free(task_xstate_cachep, ti->task->thread.xstate); - ti->task->thread.xstate = NULL; + if (tsk->thread.xstate) { + kmem_cache_free(task_xstate_cachep, tsk->thread.xstate); + tsk->thread.xstate = NULL; + } +} + +void free_thread_info(struct thread_info *ti) +{ + free_thread_xstate(ti->task); free_pages((unsigned long)(ti), get_order(THREAD_SIZE)); } diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 3890a5dd25f9..7adad088e373 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -521,6 +521,10 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) regs->cs = __USER_CS; regs->ip = new_ip; regs->sp = new_sp; + /* + * Free the old FP and other extended state + */ + free_thread_xstate(current); } EXPORT_SYMBOL_GPL(start_thread); diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index b795e831afd6..891af1a1b48a 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -533,6 +533,10 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) regs->ss = __USER_DS; regs->flags = 0x200; set_fs(USER_DS); + /* + * Free the old FP and other extended state + */ + free_thread_xstate(current); } EXPORT_SYMBOL_GPL(start_thread); diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c index 8d136a73ce8e..471e694d6713 100644 --- a/arch/x86/kernel/traps_32.c +++ b/arch/x86/kernel/traps_32.c @@ -1148,9 +1148,22 @@ asmlinkage void math_state_restore(void) struct thread_info *thread = current_thread_info(); struct task_struct *tsk = thread->task; + if (!tsk_used_math(tsk)) { + local_irq_enable(); + /* + * does a slab alloc which can sleep + */ + if (init_fpu(tsk)) { + /* + * ran out of memory! + */ + do_group_exit(SIGKILL); + return; + } + local_irq_disable(); + } + clts(); /* Allow maths ops (or we recurse) */ - if (!tsk_used_math(tsk)) - init_fpu(tsk); restore_fpu(tsk); thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */ tsk->fpu_counter++; diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c index dc0cb497eec3..adff76ea97c4 100644 --- a/arch/x86/kernel/traps_64.c +++ b/arch/x86/kernel/traps_64.c @@ -1124,10 +1124,23 @@ asmlinkage void __attribute__((weak)) mce_threshold_interrupt(void) asmlinkage void math_state_restore(void) { struct task_struct *me = current; - clts(); /* Allow maths ops (or we recurse) */ - if (!used_math()) - init_fpu(me); + if (!used_math()) { + local_irq_enable(); + /* + * does a slab alloc which can sleep + */ + if (init_fpu(me)) { + /* + * ran out of memory! + */ + do_group_exit(SIGKILL); + return; + } + local_irq_disable(); + } + + clts(); /* Allow maths ops (or we recurse) */ restore_fpu_checking(&me->thread.xstate->fxsave); task_thread_info(me)->status |= TS_USEDFPU; me->fpu_counter++; diff --git a/include/asm-x86/i387.h b/include/asm-x86/i387.h index 382a5fa9d492..4be7b58b1e16 100644 --- a/include/asm-x86/i387.h +++ b/include/asm-x86/i387.h @@ -21,7 +21,7 @@ extern void fpu_init(void); extern void mxcsr_feature_mask_init(void); -extern void init_fpu(struct task_struct *child); +extern int init_fpu(struct task_struct *child); extern asmlinkage void math_state_restore(void); extern void init_thread_xstate(void); diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index 99d297885780..e6bf92ddeb21 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -366,6 +366,8 @@ DECLARE_PER_CPU(struct orig_ist, orig_ist); extern void print_cpu_info(struct cpuinfo_x86 *); extern unsigned int xstate_size; +extern void free_thread_xstate(struct task_struct *); +extern struct kmem_cache *task_xstate_cachep; extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); extern unsigned short num_cache_leaves; -- cgit v1.2.3 From 1679f2710ac58df580d3716fab1f42ae50a226eb Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Wed, 16 Apr 2008 10:27:53 +0200 Subject: x86: fpu xstate split cleanup Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/process.c | 3 +-- include/asm-x86/i387.h | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 0e613e7e7b5e..3004d716539d 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -29,11 +29,10 @@ void free_thread_xstate(struct task_struct *tsk) } } - void free_thread_info(struct thread_info *ti) { free_thread_xstate(ti->task); - free_pages((unsigned long)(ti), get_order(THREAD_SIZE)); + free_pages((unsigned long)ti, get_order(THREAD_SIZE)); } void arch_task_cache_init(void) diff --git a/include/asm-x86/i387.h b/include/asm-x86/i387.h index 4be7b58b1e16..da2adb45f6e3 100644 --- a/include/asm-x86/i387.h +++ b/include/asm-x86/i387.h @@ -328,7 +328,7 @@ static inline unsigned short get_fpu_cwd(struct task_struct *tsk) if (cpu_has_fxsr) { return tsk->thread.xstate->fxsave.cwd; } else { - return (unsigned short) tsk->thread.xstate->fsave.cwd; + return (unsigned short)tsk->thread.xstate->fsave.cwd; } } @@ -337,7 +337,7 @@ static inline unsigned short get_fpu_swd(struct task_struct *tsk) if (cpu_has_fxsr) { return tsk->thread.xstate->fxsave.swd; } else { - return (unsigned short) tsk->thread.xstate->fsave.swd; + return (unsigned short)tsk->thread.xstate->fsave.swd; } } -- cgit v1.2.3 From 2adee9b30d1382fba97825b9c50e4f50a0117c36 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Wed, 16 Apr 2008 10:25:35 +0200 Subject: x86: fpu xstate split fix Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/thread_info.h | 1 + kernel/fork.c | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-x86/thread_info.h b/include/asm-x86/thread_info.h index 407b88c170d3..77244f17993f 100644 --- a/include/asm-x86/thread_info.h +++ b/include/asm-x86/thread_info.h @@ -9,5 +9,6 @@ extern void arch_task_cache_init(void); extern void free_thread_info(struct thread_info *ti); extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); +#define arch_task_cache_init arch_task_cache_init #endif #endif /* _ASM_X86_THREAD_INFO_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 44a18192c420..89fe414645e9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -132,9 +132,13 @@ void __put_task_struct(struct task_struct *tsk) free_task(tsk); } -void __attribute__((weak)) arch_task_cache_init(void) -{ -} +/* + * macro override instead of weak attribute alias, to workaround + * gcc 4.1.0 and 4.1.1 bugs with weak attribute and empty functions. + */ +#ifndef arch_task_cache_init +#define arch_task_cache_init() +#endif void __init fork_init(unsigned long mempages) { -- cgit v1.2.3 From 6ec6e0d9f2fd7cb6ca6bc3bfab5ae7b5cdd8c36f Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Tue, 25 Mar 2008 10:14:35 -0700 Subject: srat, x86: add support for nodes spanning other nodes For example, If the physical address layout on a two node system with 8 GB memory is something like: node 0: 0-2GB, 4-6GB node 1: 2-4GB, 6-8GB Current kernels fail to boot/detect this NUMA topology. ACPI SRAT tables can expose such a topology which needs to be supported. Signed-off-by: Suresh Siddha Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/Kconfig | 9 +++++++++ arch/x86/mm/k8topology_64.c | 2 +- arch/x86/mm/numa_64.c | 16 +++++++++++----- arch/x86/mm/srat_64.c | 32 +++++++++++++++++++++----------- include/asm-x86/numa_64.h | 3 ++- 5 files changed, 44 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2a59dbb28248..07cf77113565 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -903,6 +903,15 @@ config X86_64_ACPI_NUMA help Enable ACPI SRAT based node topology detection. +# Some NUMA nodes have memory ranges that span +# other nodes. Even though a pfn is valid and +# between a node's start and end pfns, it may not +# reside on that node. See memmap_init_zone() +# for details. +config NODES_SPAN_OTHER_NODES + def_bool y + depends on X86_64_ACPI_NUMA + config NUMA_EMU bool "NUMA emulation" depends on X86_64 && NUMA diff --git a/arch/x86/mm/k8topology_64.c b/arch/x86/mm/k8topology_64.c index 7a2ebce87df5..86808e666f9c 100644 --- a/arch/x86/mm/k8topology_64.c +++ b/arch/x86/mm/k8topology_64.c @@ -164,7 +164,7 @@ int __init k8_scan_nodes(unsigned long start, unsigned long end) if (!found) return -1; - memnode_shift = compute_hash_shift(nodes, 8); + memnode_shift = compute_hash_shift(nodes, 8, NULL); if (memnode_shift < 0) { printk(KERN_ERR "No NUMA node hash function found. Contact maintainer\n"); return -1; diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index 2ea56f48f29b..cb3170186355 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c @@ -60,7 +60,7 @@ unsigned long __initdata nodemap_size; * -1 if node overlap or lost ram (shift too big) */ static int __init populate_memnodemap(const struct bootnode *nodes, - int numnodes, int shift) + int numnodes, int shift, int *nodeids) { unsigned long addr, end; int i, res = -1; @@ -76,7 +76,12 @@ static int __init populate_memnodemap(const struct bootnode *nodes, do { if (memnodemap[addr >> shift] != NUMA_NO_NODE) return -1; - memnodemap[addr >> shift] = i; + + if (!nodeids) + memnodemap[addr >> shift] = i; + else + memnodemap[addr >> shift] = nodeids[i]; + addr += (1UL << shift); } while (addr < end); res = 1; @@ -139,7 +144,8 @@ static int __init extract_lsb_from_nodes(const struct bootnode *nodes, return i; } -int __init compute_hash_shift(struct bootnode *nodes, int numnodes) +int __init compute_hash_shift(struct bootnode *nodes, int numnodes, + int *nodeids) { int shift; @@ -149,7 +155,7 @@ int __init compute_hash_shift(struct bootnode *nodes, int numnodes) printk(KERN_DEBUG "NUMA: Using %d for the hash shift.\n", shift); - if (populate_memnodemap(nodes, numnodes, shift) != 1) { + if (populate_memnodemap(nodes, numnodes, shift, nodeids) != 1) { printk(KERN_INFO "Your memory is not aligned you need to " "rebuild your kernel with a bigger NODEMAPSIZE " "shift=%d\n", shift); @@ -462,7 +468,7 @@ done: } } out: - memnode_shift = compute_hash_shift(nodes, num_nodes); + memnode_shift = compute_hash_shift(nodes, num_nodes, NULL); if (memnode_shift < 0) { memnode_shift = 0; printk(KERN_ERR "No NUMA hash function found. NUMA emulation " diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c index 1bae9c855ceb..fb43d89f46f3 100644 --- a/arch/x86/mm/srat_64.c +++ b/arch/x86/mm/srat_64.c @@ -32,6 +32,10 @@ static struct bootnode nodes_add[MAX_NUMNODES]; static int found_add_area __initdata; int hotadd_percent __initdata = 0; +static int num_node_memblks __initdata; +static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata; +static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata; + /* Too small nodes confuse the VM badly. Usually they result from BIOS bugs. */ #define NODE_MIN_SIZE (4*1024*1024) @@ -41,17 +45,17 @@ static __init int setup_node(int pxm) return acpi_map_pxm_to_node(pxm); } -static __init int conflicting_nodes(unsigned long start, unsigned long end) +static __init int conflicting_memblks(unsigned long start, unsigned long end) { int i; - for_each_node_mask(i, nodes_parsed) { - struct bootnode *nd = &nodes[i]; + for (i = 0; i < num_node_memblks; i++) { + struct bootnode *nd = &node_memblk_range[i]; if (nd->start == nd->end) continue; if (nd->end > start && nd->start < end) - return i; + return memblk_nodeid[i]; if (nd->end == end && nd->start == start) - return i; + return memblk_nodeid[i]; } return -1; } @@ -258,7 +262,7 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) bad_srat(); return; } - i = conflicting_nodes(start, end); + i = conflicting_memblks(start, end); if (i == node) { printk(KERN_WARNING "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n", @@ -283,10 +287,10 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) nd->end = end; } - printk(KERN_INFO "SRAT: Node %u PXM %u %Lx-%Lx\n", node, pxm, - nd->start, nd->end); - e820_register_active_regions(node, nd->start >> PAGE_SHIFT, - nd->end >> PAGE_SHIFT); + printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm, + start, end); + e820_register_active_regions(node, start >> PAGE_SHIFT, + end >> PAGE_SHIFT); push_node_boundaries(node, nd->start >> PAGE_SHIFT, nd->end >> PAGE_SHIFT); @@ -298,6 +302,11 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) if ((nd->start | nd->end) == 0) node_clear(node, nodes_parsed); } + + node_memblk_range[num_node_memblks].start = start; + node_memblk_range[num_node_memblks].end = end; + memblk_nodeid[num_node_memblks] = node; + num_node_memblks++; } /* Sanity check to catch more bad SRATs (they are amazingly common). @@ -368,7 +377,8 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) return -1; } - memnode_shift = compute_hash_shift(nodes, MAX_NUMNODES); + memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks, + memblk_nodeid); if (memnode_shift < 0) { printk(KERN_ERR "SRAT: No NUMA node hash function found. Contact maintainer\n"); diff --git a/include/asm-x86/numa_64.h b/include/asm-x86/numa_64.h index 32c22ae0709f..22e87c9f6a80 100644 --- a/include/asm-x86/numa_64.h +++ b/include/asm-x86/numa_64.h @@ -9,7 +9,8 @@ struct bootnode { u64 end; }; -extern int compute_hash_shift(struct bootnode *nodes, int numnodes); +extern int compute_hash_shift(struct bootnode *nodes, int numblks, + int *nodeids); #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT)) -- cgit v1.2.3 From 752bea4abbff5e3ffef36802b860e80d0b632990 Mon Sep 17 00:00:00 2001 From: Yinghai Lu Date: Fri, 7 Mar 2008 15:02:50 -0800 Subject: x86: reserve dma32 early for gart a system with 256 GB of RAM, when NUMA is disabled crashes the following way: Your BIOS doesn't leave a aperture memory hole Please enable the IOMMU option in the BIOS setup This costs you 64 MB of RAM Cannot allocate aperture memory hole (ffff8101c0000000,65536K) Kernel panic - not syncing: Not enough memory for aperture Pid: 0, comm: swapper Not tainted 2.6.25-rc4-x86-latest.git #33 Call Trace: [] panic+0xb2/0x190 [] ? release_console_sem+0x7c/0x250 [] ? __alloc_bootmem_nopanic+0x48/0x90 [] ? free_bootmem+0x29/0x50 [] gart_iommu_hole_init+0x5e7/0x680 [] ? alloc_large_system_hash+0x16b/0x310 [] ? _etext+0x0/0x1 [] pci_iommu_alloc+0x1c/0x40 [] mem_init+0x45/0x1a0 [] start_kernel+0x295/0x380 [] _sinittext+0x1c2/0x230 the root cause is : memmap PMD is too big, [ffffe200e0600000-ffffe200e07fffff] PMD ->ffff81383c000000 on node 0 almost near 4G..., and vmemmap_alloc_block will use up the ram under 4G. solution will be: 1. make memmap allocation get memory above 4G... 2. reserve some dma32 range early before we try to set up memmap for all. and release that before pci_iommu_alloc, so gart or swiotlb could get some range under 4g limit for sure. the patch is using method 2. because method1 may need more code to handle SPARSEMEM and SPASEMEM_VMEMMAP will get Your BIOS doesn't leave a aperture memory hole Please enable the IOMMU option in the BIOS setup This costs you 64 MB of RAM Mapping aperture over 65536 KB of RAM @ 4000000 Memory: 264245736k/268959744k available (8484k kernel code, 4187464k reserved, 4004k data, 724k init) Signed-off-by: Yinghai Lu Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-dma_64.c | 49 ++++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/setup_64.c | 2 ++ include/asm-x86/pci_64.h | 1 + 3 files changed, 52 insertions(+) (limited to 'include') diff --git a/arch/x86/kernel/pci-dma_64.c b/arch/x86/kernel/pci-dma_64.c index ada5a0604992..e4fffaabe53b 100644 --- a/arch/x86/kernel/pci-dma_64.c +++ b/arch/x86/kernel/pci-dma_64.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -286,8 +288,55 @@ static __init int iommu_setup(char *p) } early_param("iommu", iommu_setup); +static __initdata void *dma32_bootmem_ptr; +static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); + +static int __init parse_dma32_size_opt(char *p) +{ + if (!p) + return -EINVAL; + dma32_bootmem_size = memparse(p, &p); + return 0; +} +early_param("dma32_size", parse_dma32_size_opt); + +void __init dma32_reserve_bootmem(void) +{ + unsigned long size, align; + if (end_pfn <= MAX_DMA32_PFN) + return; + + align = 64ULL<<20; + size = round_up(dma32_bootmem_size, align); + dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, + __pa(MAX_DMA_ADDRESS)); + if (dma32_bootmem_ptr) + dma32_bootmem_size = size; + else + dma32_bootmem_size = 0; +} +static void __init dma32_free_bootmem(void) +{ + int node; + + if (end_pfn <= MAX_DMA32_PFN) + return; + + if (!dma32_bootmem_ptr) + return; + + for_each_online_node(node) + free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr), + dma32_bootmem_size); + + dma32_bootmem_ptr = NULL; + dma32_bootmem_size = 0; +} + void __init pci_iommu_alloc(void) { + /* free the range so iommu could get some range less than 4G */ + dma32_free_bootmem(); /* * The order of these functions is important for * fall-back/fail-over reasons diff --git a/arch/x86/kernel/setup_64.c b/arch/x86/kernel/setup_64.c index 0aa291bff4e0..6b8e11f0c15d 100644 --- a/arch/x86/kernel/setup_64.c +++ b/arch/x86/kernel/setup_64.c @@ -398,6 +398,8 @@ void __init setup_arch(char **cmdline_p) early_res_to_bootmem(); + dma32_reserve_bootmem(); + #ifdef CONFIG_ACPI_SLEEP /* * Reserve low memory region for sleep support. diff --git a/include/asm-x86/pci_64.h b/include/asm-x86/pci_64.h index df867e5d80b1..f330234ffa5c 100644 --- a/include/asm-x86/pci_64.h +++ b/include/asm-x86/pci_64.h @@ -22,6 +22,7 @@ extern int (*pci_config_read)(int seg, int bus, int dev, int fn, extern int (*pci_config_write)(int seg, int bus, int dev, int fn, int reg, int len, u32 value); +extern void dma32_reserve_bootmem(void); extern void pci_iommu_alloc(void); /* The PCI address space does equal the physical memory -- cgit v1.2.3 From 6f5366354bf86f8d2c1cf241c9bbf44b2d350e30 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:20 -0300 Subject: x86: move dma_ops struct definition to dma-mapping.h take it off the x86_64 specific header Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 54 ++++++++++++++++++++++++++++++++++++++++ include/asm-x86/dma-mapping_64.h | 49 ------------------------------------ 2 files changed, 54 insertions(+), 49 deletions(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 58f790f4df52..aebd178a19ac 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -1,5 +1,59 @@ +#ifndef _ASM_DMA_MAPPING_H_ +#define _ASM_DMA_MAPPING_H_ + +/* + * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for + * documentation. + */ + +#include +#include +#include + +struct dma_mapping_ops { + int (*mapping_error)(dma_addr_t dma_addr); + void* (*alloc_coherent)(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp); + void (*free_coherent)(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle); + dma_addr_t (*map_single)(struct device *hwdev, void *ptr, + size_t size, int direction); + /* like map_single, but doesn't check the device mask */ + dma_addr_t (*map_simple)(struct device *hwdev, char *ptr, + size_t size, int direction); + void (*unmap_single)(struct device *dev, dma_addr_t addr, + size_t size, int direction); + void (*sync_single_for_cpu)(struct device *hwdev, + dma_addr_t dma_handle, size_t size, + int direction); + void (*sync_single_for_device)(struct device *hwdev, + dma_addr_t dma_handle, size_t size, + int direction); + void (*sync_single_range_for_cpu)(struct device *hwdev, + dma_addr_t dma_handle, unsigned long offset, + size_t size, int direction); + void (*sync_single_range_for_device)(struct device *hwdev, + dma_addr_t dma_handle, unsigned long offset, + size_t size, int direction); + void (*sync_sg_for_cpu)(struct device *hwdev, + struct scatterlist *sg, int nelems, + int direction); + void (*sync_sg_for_device)(struct device *hwdev, + struct scatterlist *sg, int nelems, + int direction); + int (*map_sg)(struct device *hwdev, struct scatterlist *sg, + int nents, int direction); + void (*unmap_sg)(struct device *hwdev, + struct scatterlist *sg, int nents, + int direction); + int (*dma_supported)(struct device *hwdev, u64 mask); + int is_phys; +}; + #ifdef CONFIG_X86_32 # include "dma-mapping_32.h" #else # include "dma-mapping_64.h" #endif + +#endif diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index ecd0f6125ba3..369188a348f4 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -1,55 +1,6 @@ #ifndef _X8664_DMA_MAPPING_H #define _X8664_DMA_MAPPING_H 1 -/* - * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for - * documentation. - */ - -#include -#include -#include - -struct dma_mapping_ops { - int (*mapping_error)(dma_addr_t dma_addr); - void* (*alloc_coherent)(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp); - void (*free_coherent)(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle); - dma_addr_t (*map_single)(struct device *hwdev, void *ptr, - size_t size, int direction); - /* like map_single, but doesn't check the device mask */ - dma_addr_t (*map_simple)(struct device *hwdev, char *ptr, - size_t size, int direction); - void (*unmap_single)(struct device *dev, dma_addr_t addr, - size_t size, int direction); - void (*sync_single_for_cpu)(struct device *hwdev, - dma_addr_t dma_handle, size_t size, - int direction); - void (*sync_single_for_device)(struct device *hwdev, - dma_addr_t dma_handle, size_t size, - int direction); - void (*sync_single_range_for_cpu)(struct device *hwdev, - dma_addr_t dma_handle, unsigned long offset, - size_t size, int direction); - void (*sync_single_range_for_device)(struct device *hwdev, - dma_addr_t dma_handle, unsigned long offset, - size_t size, int direction); - void (*sync_sg_for_cpu)(struct device *hwdev, - struct scatterlist *sg, int nelems, - int direction); - void (*sync_sg_for_device)(struct device *hwdev, - struct scatterlist *sg, int nelems, - int direction); - int (*map_sg)(struct device *hwdev, struct scatterlist *sg, - int nents, int direction); - void (*unmap_sg)(struct device *hwdev, - struct scatterlist *sg, int nents, - int direction); - int (*dma_supported)(struct device *hwdev, u64 mask); - int is_phys; -}; - extern dma_addr_t bad_dma_address; extern const struct dma_mapping_ops* dma_ops; extern int iommu_merge; -- cgit v1.2.3 From 22456b97148be300e25e9cb97244656775972475 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:21 -0300 Subject: x86: implement dma_map_single through dma_ops That's already the name of the game for x86_64. For i386, we add a pci-base_32.c, that will hold the default operations. The function call itself goes through dma-mapping.h , the common header Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/Makefile | 1 + arch/x86/kernel/pci-base_32.c | 20 ++++++++++++++++++++ include/asm-x86/dma-mapping.h | 10 ++++++++++ include/asm-x86/dma-mapping_32.h | 10 ---------- include/asm-x86/dma-mapping_64.h | 9 --------- 5 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 arch/x86/kernel/pci-base_32.c (limited to 'include') diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 7a2a2e93e84b..edd5c54ffde9 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -26,6 +26,7 @@ obj-y += pci-dma_$(BITS).o bootflag.o e820_$(BITS).o obj-y += quirks.o i8237.o topology.o kdebugfs.o obj-y += alternative.o i8253.o obj-$(CONFIG_X86_64) += pci-nommu_64.o bugs_64.o +obj-$(CONFIG_X86_32) += pci-base_32.o obj-y += tsc_$(BITS).o io_delay.o rtc.o obj-$(CONFIG_X86_TRAMPOLINE) += trampoline.o diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c new file mode 100644 index 000000000000..b613d735f76c --- /dev/null +++ b/arch/x86/kernel/pci-base_32.c @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include + +static dma_addr_t pci32_map_single(struct device *dev, void *ptr, + size_t size, int direction) +{ + WARN_ON(size == 0); + flush_write_buffers(); + return virt_to_phys(ptr); +} + +static const struct dma_mapping_ops pci32_dma_ops = { + .map_single = pci32_map_single, +}; + +const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; +EXPORT_SYMBOL(dma_ops); diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index aebd178a19ac..d320244db8af 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -50,10 +50,20 @@ struct dma_mapping_ops { int is_phys; }; +extern const struct dma_mapping_ops *dma_ops; + #ifdef CONFIG_X86_32 # include "dma-mapping_32.h" #else # include "dma-mapping_64.h" #endif +static inline dma_addr_t +dma_map_single(struct device *hwdev, void *ptr, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_single(hwdev, ptr, size, direction); +} + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index 55f01bd9e556..b496306d5e98 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -17,16 +17,6 @@ void *dma_alloc_coherent(struct device *dev, size_t size, void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline dma_addr_t -dma_map_single(struct device *dev, void *ptr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); - WARN_ON(size == 0); - flush_write_buffers(); - return virt_to_phys(ptr); -} - static inline void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 369188a348f4..969a7da0cf97 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -2,7 +2,6 @@ #define _X8664_DMA_MAPPING_H 1 extern dma_addr_t bad_dma_address; -extern const struct dma_mapping_ops* dma_ops; extern int iommu_merge; static inline int dma_mapping_error(dma_addr_t dma_addr) @@ -24,14 +23,6 @@ extern void *dma_alloc_coherent(struct device *dev, size_t size, extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline dma_addr_t -dma_map_single(struct device *hwdev, void *ptr, size_t size, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return dma_ops->map_single(hwdev, ptr, size, direction); -} - static inline void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size, int direction) -- cgit v1.2.3 From 0cb0ae68323657663e4e8c0c1ce82a5af6621bbb Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:22 -0300 Subject: x86: move dma_unmap_single to common header i386 base does not need it, so it gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 10 ++++++++++ include/asm-x86/dma-mapping_32.h | 7 ------- include/asm-x86/dma-mapping_64.h | 8 -------- 4 files changed, 11 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index b613d735f76c..a8a7c7f2d23f 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -14,6 +14,7 @@ static dma_addr_t pci32_map_single(struct device *dev, void *ptr, static const struct dma_mapping_ops pci32_dma_ops = { .map_single = pci32_map_single, + .unmap_single = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index d320244db8af..bb0378f2b933 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -66,4 +66,14 @@ dma_map_single(struct device *hwdev, void *ptr, size_t size, return dma_ops->map_single(hwdev, ptr, size, direction); } +static inline void +dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->unmap_single) + dma_ops->unmap_single(dev, addr, size, direction); +} + + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index b496306d5e98..0b27cb0dbb75 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -17,13 +17,6 @@ void *dma_alloc_coherent(struct device *dev, size_t size, void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline void -dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - static inline int dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 969a7da0cf97..5d349db23c99 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -23,14 +23,6 @@ extern void *dma_alloc_coherent(struct device *dev, size_t size, extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline void -dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - dma_ops->unmap_single(dev, addr, size, direction); -} - #define dma_map_page(dev,page,offset,size,dir) \ dma_map_single((dev), page_address(page)+(offset), (size), (dir)) -- cgit v1.2.3 From 16a3ce9bae667178f79a4951fc0ba8b515b5b733 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:23 -0300 Subject: x86: move dma_map_sg to common header the old i386 implementation is moved to pci-base_32.c Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 19 +++++++++++++++++++ include/asm-x86/dma-mapping.h | 8 +++++++- include/asm-x86/dma-mapping_32.h | 20 -------------------- include/asm-x86/dma-mapping_64.h | 7 ------- 4 files changed, 26 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index a8a7c7f2d23f..24741525901c 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -12,9 +12,28 @@ static dma_addr_t pci32_map_single(struct device *dev, void *ptr, return virt_to_phys(ptr); } +static int pci32_dma_map_sg(struct device *dev, struct scatterlist *sglist, + int nents, int direction) +{ + struct scatterlist *sg; + int i; + + WARN_ON(nents == 0 || sglist[0].length == 0); + + for_each_sg(sglist, sg, nents, i) { + BUG_ON(!sg_page(sg)); + + sg->dma_address = sg_phys(sg); + } + + flush_write_buffers(); + return nents; +} + static const struct dma_mapping_ops pci32_dma_ops = { .map_single = pci32_map_single, .unmap_single = NULL, + .map_sg = pci32_dma_map_sg, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index bb0378f2b933..09011546a48b 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -75,5 +75,11 @@ dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, dma_ops->unmap_single(dev, addr, size, direction); } - +static inline int +dma_map_sg(struct device *hwdev, struct scatterlist *sg, + int nents, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_sg(hwdev, sg, nents, direction); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index 0b27cb0dbb75..cdcdeff1e597 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -17,26 +17,6 @@ void *dma_alloc_coherent(struct device *dev, size_t size, void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline int -dma_map_sg(struct device *dev, struct scatterlist *sglist, int nents, - enum dma_data_direction direction) -{ - struct scatterlist *sg; - int i; - - BUG_ON(!valid_dma_direction(direction)); - WARN_ON(nents == 0 || sglist[0].length == 0); - - for_each_sg(sglist, sg, nents, i) { - BUG_ON(!sg_page(sg)); - - sg->dma_address = sg_phys(sg); - } - - flush_write_buffers(); - return nents; -} - static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 5d349db23c99..b27527ab770e 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -96,13 +96,6 @@ dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, flush_write_buffers(); } -static inline int -dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return dma_ops->map_sg(hwdev, sg, nents, direction); -} - static inline void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, int direction) -- cgit v1.2.3 From 72c784f82c378df1903676acd2efc5eeb5cac579 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:24 -0300 Subject: x86: move dma_unmap_sg to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 9 +++++++++ include/asm-x86/dma-mapping_32.h | 8 -------- include/asm-x86/dma-mapping_64.h | 8 -------- 4 files changed, 10 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 24741525901c..920530438d8c 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -34,6 +34,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .map_single = pci32_map_single, .unmap_single = NULL, .map_sg = pci32_dma_map_sg, + .unmap_sg = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 09011546a48b..6e7747a23e5b 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -82,4 +82,13 @@ dma_map_sg(struct device *hwdev, struct scatterlist *sg, BUG_ON(!valid_dma_direction(direction)); return dma_ops->map_sg(hwdev, sg, nents, direction); } + +static inline void +dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->unmap_sg) + dma_ops->unmap_sg(hwdev, sg, nents, direction); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index cdcdeff1e597..55445e3e2d66 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,14 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } - -static inline void -dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index b27527ab770e..ce37efb847b3 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -96,14 +96,6 @@ dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, flush_write_buffers(); } -static inline void -dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, - int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - dma_ops->unmap_sg(hwdev, sg, nents, direction); -} - extern int dma_supported(struct device *hwdev, u64 mask); /* same for gart, swiotlb, and nommu */ -- cgit v1.2.3 From c01dd8cf7d19b869af1668c80a34a955c871f607 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:25 -0300 Subject: x86: move dma_sync_single_for_cpu to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 12 ++++++++++++ include/asm-x86/dma-mapping_32.h | 6 ------ include/asm-x86/dma-mapping_64.h | 11 ----------- 4 files changed, 13 insertions(+), 17 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 920530438d8c..dce03c81bb25 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -35,6 +35,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .unmap_single = NULL, .map_sg = pci32_dma_map_sg, .unmap_sg = NULL, + .sync_single_for_cpu = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 6e7747a23e5b..507069d231dd 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -91,4 +91,16 @@ dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, if (dma_ops->unmap_sg) dma_ops->unmap_sg(hwdev, sg, nents, direction); } + +static inline void +dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, + size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_for_cpu) + dma_ops->sync_single_for_cpu(hwdev, dma_handle, size, + direction); + flush_write_buffers(); +} + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index 55445e3e2d66..a05b2fc810bb 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,12 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ -} - static inline void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index ce37efb847b3..b055964d1a27 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -28,17 +28,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, #define dma_unmap_page dma_unmap_single -static inline void -dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_for_cpu) - dma_ops->sync_single_for_cpu(hwdev, dma_handle, size, - direction); - flush_write_buffers(); -} - static inline void dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, size_t size, int direction) -- cgit v1.2.3 From 9231b269e09ed60910c159cf668f887623b7ac58 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:26 -0300 Subject: x86: move dma_sync_single_for_device to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 11 +++++++++++ include/asm-x86/dma-mapping_32.h | 7 ------- include/asm-x86/dma-mapping_64.h | 11 ----------- 4 files changed, 12 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index dce03c81bb25..36488245e361 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -36,6 +36,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .map_sg = pci32_dma_map_sg, .unmap_sg = NULL, .sync_single_for_cpu = NULL, + .sync_single_for_device = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 507069d231dd..8ad582c2080a 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -103,4 +103,15 @@ dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, flush_write_buffers(); } +static inline void +dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, + size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_for_device) + dma_ops->sync_single_for_device(hwdev, dma_handle, size, + direction); + flush_write_buffers(); +} + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index a05b2fc810bb..b91771a6a0bd 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,13 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - static inline void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index b055964d1a27..b539f61f3c78 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -28,17 +28,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, #define dma_unmap_page dma_unmap_single -static inline void -dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, - size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_for_device) - dma_ops->sync_single_for_device(hwdev, dma_handle, size, - direction); - flush_write_buffers(); -} - static inline void dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) -- cgit v1.2.3 From 627610fcb70164991ed0d11110a56c43b15b9312 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:27 -0300 Subject: x86: move dma_sync_single_range_for_cpu to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 11 +++++++++++ include/asm-x86/dma-mapping_32.h | 7 ------- include/asm-x86/dma-mapping_64.h | 12 ------------ 4 files changed, 12 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 36488245e361..c501599a4506 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -37,6 +37,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .unmap_sg = NULL, .sync_single_for_cpu = NULL, .sync_single_for_device = NULL, + .sync_single_range_for_cpu = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 8ad582c2080a..a466470f130d 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -114,4 +114,15 @@ dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, flush_write_buffers(); } +static inline void +dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, + unsigned long offset, size_t size, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_range_for_cpu) + dma_ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, + size, direction); + + flush_write_buffers(); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index b91771a6a0bd..e24c59d76ecb 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,13 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, - enum dma_data_direction direction) -{ -} - static inline void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index b539f61f3c78..6ecafad1554b 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -28,18 +28,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, #define dma_unmap_page dma_unmap_single -static inline void -dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_range_for_cpu) { - dma_ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, size, direction); - } - - flush_write_buffers(); -} - static inline void dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, unsigned long offset, size_t size, int direction) -- cgit v1.2.3 From 713623326c816b145105769f174ec237815e53f1 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:28 -0300 Subject: x86: move dma_sync_single_range_for_device to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 14 ++++++++++++++ include/asm-x86/dma-mapping_32.h | 8 -------- include/asm-x86/dma-mapping_64.h | 12 ------------ 4 files changed, 15 insertions(+), 20 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index c501599a4506..4512c307b60c 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -38,6 +38,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .sync_single_for_cpu = NULL, .sync_single_for_device = NULL, .sync_single_range_for_cpu = NULL, + .sync_single_range_for_device = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index a466470f130d..260538b6ce2c 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -125,4 +125,18 @@ dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, flush_write_buffers(); } + +static inline void +dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, + unsigned long offset, size_t size, + int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_single_range_for_device) + dma_ops->sync_single_range_for_device(hwdev, dma_handle, + offset, size, direction); + + flush_write_buffers(); +} + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index e24c59d76ecb..60d5371c7de7 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,14 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, - unsigned long offset, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - static inline void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 6ecafad1554b..dfa66cc525bc 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -28,18 +28,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, #define dma_unmap_page dma_unmap_single -static inline void -dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, - unsigned long offset, size_t size, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_single_range_for_device) - dma_ops->sync_single_range_for_device(hwdev, dma_handle, - offset, size, direction); - - flush_write_buffers(); -} - static inline void dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, int nelems, int direction) -- cgit v1.2.3 From ed435dee9cb470082e4550edbfcbc7e81132e976 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:29 -0300 Subject: x86: move dma_sync_sg_for_cpu to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 9 +++++++++ include/asm-x86/dma-mapping_32.h | 6 ------ include/asm-x86/dma-mapping_64.h | 11 ----------- 4 files changed, 10 insertions(+), 17 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 4512c307b60c..d876600aaeb6 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -39,6 +39,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .sync_single_for_device = NULL, .sync_single_range_for_cpu = NULL, .sync_single_range_for_device = NULL, + .sync_sg_for_cpu = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 260538b6ce2c..1a301d7e4726 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -139,4 +139,13 @@ dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, flush_write_buffers(); } +static inline void +dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, + int nelems, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_sg_for_cpu) + dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); + flush_write_buffers(); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index 60d5371c7de7..e7c82e34af9e 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,12 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, - enum dma_data_direction direction) -{ -} - static inline void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index dfa66cc525bc..804b154abaf8 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -27,17 +27,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_map_single((dev), page_address(page)+(offset), (size), (dir)) #define dma_unmap_page dma_unmap_single - -static inline void -dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_sg_for_cpu) - dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); - flush_write_buffers(); -} - static inline void dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, int nelems, int direction) -- cgit v1.2.3 From e7f3a913f91b7bfef3a93dff27930f24bdfcd2c0 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:30 -0300 Subject: x86: move dma_sync_sg_for_device to common header i386 gets an empty function. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/dma-mapping.h | 11 +++++++++++ include/asm-x86/dma-mapping_32.h | 7 ------- include/asm-x86/dma-mapping_64.h | 12 ------------ 4 files changed, 12 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index d876600aaeb6..033d94ec5000 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -40,6 +40,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .sync_single_range_for_cpu = NULL, .sync_single_range_for_device = NULL, .sync_sg_for_cpu = NULL, + .sync_sg_for_device = NULL, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 1a301d7e4726..53a404b0f9ee 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -148,4 +148,15 @@ dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); flush_write_buffers(); } + +static inline void +dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, + int nelems, int direction) +{ + BUG_ON(!valid_dma_direction(direction)); + if (dma_ops->sync_sg_for_device) + dma_ops->sync_sg_for_device(hwdev, sg, nelems, direction); + + flush_write_buffers(); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index e7c82e34af9e..eff42f495107 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -32,13 +32,6 @@ dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, BUG_ON(!valid_dma_direction(direction)); } -static inline void -dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - static inline int dma_mapping_error(dma_addr_t dma_addr) { diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 804b154abaf8..707dbbe86a26 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -27,18 +27,6 @@ extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_map_single((dev), page_address(page)+(offset), (size), (dir)) #define dma_unmap_page dma_unmap_single -static inline void -dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, - int nelems, int direction) -{ - BUG_ON(!valid_dma_direction(direction)); - if (dma_ops->sync_sg_for_device) { - dma_ops->sync_sg_for_device(hwdev, sg, nelems, direction); - } - - flush_write_buffers(); -} - extern int dma_supported(struct device *hwdev, u64 mask); /* same for gart, swiotlb, and nommu */ -- cgit v1.2.3 From 8d396ded71a9b378fc3e846095e50565606f2df6 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:31 -0300 Subject: x86: move alloc and free coherent to common header they are the same between architectures. (except for the fact that x86_64 has duplicate code) move them to dma-mapping.h Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 10 ++++++++++ include/asm-x86/dma-mapping_32.h | 9 --------- include/asm-x86/dma-mapping_64.h | 11 ----------- 3 files changed, 10 insertions(+), 20 deletions(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 53a404b0f9ee..3ea3802ff1aa 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -52,6 +52,16 @@ struct dma_mapping_ops { extern const struct dma_mapping_ops *dma_ops; +#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) +#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) + +void *dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag); + +void dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle); + + #ifdef CONFIG_X86_32 # include "dma-mapping_32.h" #else diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index eff42f495107..d8f6420d3ef1 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -8,15 +8,6 @@ #include #include -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag); - -void dma_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle); - static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction) diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 707dbbe86a26..ce881d992228 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -12,17 +12,6 @@ static inline int dma_mapping_error(dma_addr_t dma_addr) return (dma_addr == bad_dma_address); } -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -extern void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp); -extern void dma_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle); - #define dma_map_page(dev,page,offset,size,dir) \ dma_map_single((dev), page_address(page)+(offset), (size), (dir)) -- cgit v1.2.3 From 4d92fbf231fe23ec07d18820a141c573a7f5017a Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:32 -0300 Subject: x86: move dma_map_page and dma_unmap_page to common header They are similar enough to do this move. the macro version is ugly, and we use inline functions instead. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 14 ++++++++++++++ include/asm-x86/dma-mapping_32.h | 15 --------------- include/asm-x86/dma-mapping_64.h | 4 ---- 3 files changed, 14 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 3ea3802ff1aa..b5a413acac96 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -169,4 +169,18 @@ dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, flush_write_buffers(); } + +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, + size_t offset, size_t size, + int direction) +{ + return dma_map_single(dev, page_address(page)+offset, size, direction); +} + +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, + size_t size, int direction) +{ + dma_unmap_single(dev, addr, size, direction); +} + #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index d8f6420d3ef1..c61ae7ff222c 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -8,21 +8,6 @@ #include #include -static inline dma_addr_t -dma_map_page(struct device *dev, struct page *page, unsigned long offset, - size_t size, enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); - return page_to_phys(page) + offset; -} - -static inline void -dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(!valid_dma_direction(direction)); -} - static inline int dma_mapping_error(dma_addr_t dma_addr) { diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index ce881d992228..2b4a43080db9 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -12,10 +12,6 @@ static inline int dma_mapping_error(dma_addr_t dma_addr) return (dma_addr == bad_dma_address); } -#define dma_map_page(dev,page,offset,size,dir) \ - dma_map_single((dev), page_address(page)+(offset), (size), (dir)) - -#define dma_unmap_page dma_unmap_single extern int dma_supported(struct device *hwdev, u64 mask); /* same for gart, swiotlb, and nommu */ -- cgit v1.2.3 From 2be621498d461b63ca6124f86e3b9582e1a8e722 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Sat, 19 Apr 2008 19:19:56 +0200 Subject: x86: dma-ops on highmem fix Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 4 ++-- arch/x86/kernel/pci-calgary_64.c | 3 ++- arch/x86/kernel/pci-dma_64.c | 2 +- arch/x86/kernel/pci-gart_64.c | 15 +++++++-------- arch/x86/kernel/pci-nommu_64.c | 4 ++-- arch/x86/kernel/pci-swiotlb_64.c | 9 ++++++++- include/asm-x86/dma-mapping.h | 10 ++++++---- 7 files changed, 28 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 033d94ec5000..cf4bb28dfc6a 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -4,12 +4,12 @@ #include #include -static dma_addr_t pci32_map_single(struct device *dev, void *ptr, +static dma_addr_t pci32_map_single(struct device *dev, phys_addr_t ptr, size_t size, int direction) { WARN_ON(size == 0); flush_write_buffers(); - return virt_to_phys(ptr); + return ptr; } static int pci32_dma_map_sg(struct device *dev, struct scatterlist *sglist, diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c index 1b5464c2434f..adb91e4b62da 100644 --- a/arch/x86/kernel/pci-calgary_64.c +++ b/arch/x86/kernel/pci-calgary_64.c @@ -470,10 +470,11 @@ error: return 0; } -static dma_addr_t calgary_map_single(struct device *dev, void *vaddr, +static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr, size_t size, int direction) { dma_addr_t dma_handle = bad_dma_address; + void *vaddr = phys_to_virt(paddr); unsigned long uaddr; unsigned int npages; struct iommu_table *tbl = find_iommu_table(dev); diff --git a/arch/x86/kernel/pci-dma_64.c b/arch/x86/kernel/pci-dma_64.c index e4fffaabe53b..f97a08d0a8f9 100644 --- a/arch/x86/kernel/pci-dma_64.c +++ b/arch/x86/kernel/pci-dma_64.c @@ -141,7 +141,7 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, } if (dma_ops->map_simple) { - *dma_handle = dma_ops->map_simple(dev, memory, + *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory), size, PCI_DMA_BIDIRECTIONAL); if (*dma_handle != bad_dma_address) diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c index 700e4647dd30..c07455d1695f 100644 --- a/arch/x86/kernel/pci-gart_64.c +++ b/arch/x86/kernel/pci-gart_64.c @@ -264,9 +264,9 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, } static dma_addr_t -gart_map_simple(struct device *dev, char *buf, size_t size, int dir) +gart_map_simple(struct device *dev, phys_addr_t paddr, size_t size, int dir) { - dma_addr_t map = dma_map_area(dev, virt_to_bus(buf), size, dir); + dma_addr_t map = dma_map_area(dev, paddr, size, dir); flush_gart(); @@ -275,18 +275,17 @@ gart_map_simple(struct device *dev, char *buf, size_t size, int dir) /* Map a single area into the IOMMU */ static dma_addr_t -gart_map_single(struct device *dev, void *addr, size_t size, int dir) +gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir) { - unsigned long phys_mem, bus; + unsigned long bus; if (!dev) dev = &fallback_dev; - phys_mem = virt_to_phys(addr); - if (!need_iommu(dev, phys_mem, size)) - return phys_mem; + if (!need_iommu(dev, paddr, size)) + return paddr; - bus = gart_map_simple(dev, addr, size, dir); + bus = gart_map_simple(dev, paddr, size, dir); return bus; } diff --git a/arch/x86/kernel/pci-nommu_64.c b/arch/x86/kernel/pci-nommu_64.c index ab08e1832228..6e330769d017 100644 --- a/arch/x86/kernel/pci-nommu_64.c +++ b/arch/x86/kernel/pci-nommu_64.c @@ -26,10 +26,10 @@ check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size) } static dma_addr_t -nommu_map_single(struct device *hwdev, void *ptr, size_t size, +nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size, int direction) { - dma_addr_t bus = virt_to_bus(ptr); + dma_addr_t bus = paddr; if (!check_addr("map_single", hwdev, bus, size)) return bad_dma_address; return bus; diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb_64.c index 82a0a674a003..490da7f4b8d0 100644 --- a/arch/x86/kernel/pci-swiotlb_64.c +++ b/arch/x86/kernel/pci-swiotlb_64.c @@ -11,11 +11,18 @@ int swiotlb __read_mostly; +static dma_addr_t +swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size, + int direction) +{ + return swiotlb_map_single(hwdev, phys_to_virt(paddr), size, direction); +} + const struct dma_mapping_ops swiotlb_dma_ops = { .mapping_error = swiotlb_dma_mapping_error, .alloc_coherent = swiotlb_alloc_coherent, .free_coherent = swiotlb_free_coherent, - .map_single = swiotlb_map_single, + .map_single = swiotlb_map_single_phys, .unmap_single = swiotlb_unmap_single, .sync_single_for_cpu = swiotlb_sync_single_for_cpu, .sync_single_for_device = swiotlb_sync_single_for_device, diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index b5a413acac96..b331a8d3a7cf 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -16,10 +16,10 @@ struct dma_mapping_ops { dma_addr_t *dma_handle, gfp_t gfp); void (*free_coherent)(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); - dma_addr_t (*map_single)(struct device *hwdev, void *ptr, + dma_addr_t (*map_single)(struct device *hwdev, phys_addr_t ptr, size_t size, int direction); /* like map_single, but doesn't check the device mask */ - dma_addr_t (*map_simple)(struct device *hwdev, char *ptr, + dma_addr_t (*map_simple)(struct device *hwdev, phys_addr_t ptr, size_t size, int direction); void (*unmap_single)(struct device *dev, dma_addr_t addr, size_t size, int direction); @@ -73,7 +73,7 @@ dma_map_single(struct device *hwdev, void *ptr, size_t size, int direction) { BUG_ON(!valid_dma_direction(direction)); - return dma_ops->map_single(hwdev, ptr, size, direction); + return dma_ops->map_single(hwdev, virt_to_phys(ptr), size, direction); } static inline void @@ -174,7 +174,9 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, size_t offset, size_t size, int direction) { - return dma_map_single(dev, page_address(page)+offset, size, direction); + BUG_ON(!valid_dma_direction(direction)); + return dma_ops->map_single(dev, page_to_phys(page)+offset, + size, direction); } static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, -- cgit v1.2.3 From 3cb6a91711a682adb3aa95da2ed8d47512cc3c41 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:33 -0300 Subject: x86: move dma_cache_sync to common header they are the same in both architectures. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 6 ++++++ include/asm-x86/dma-mapping_32.h | 7 ------- include/asm-x86/dma-mapping_64.h | 7 ------- 3 files changed, 6 insertions(+), 14 deletions(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index b331a8d3a7cf..51a79d71dabd 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -185,4 +185,10 @@ static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, dma_unmap_single(dev, addr, size, direction); } +static inline void +dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction dir) +{ + flush_write_buffers(); +} #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index c61ae7ff222c..e60c30a88311 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -55,13 +55,6 @@ dma_get_cache_alignment(void) #define dma_is_consistent(d, h) (1) -static inline void -dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} - #define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY extern int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 2b4a43080db9..b1bc6ca7613d 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -24,13 +24,6 @@ static inline int dma_get_cache_alignment(void) extern int dma_set_mask(struct device *dev, u64 mask); -static inline void -dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction dir) -{ - flush_write_buffers(); -} - extern struct device fallback_dev; extern int panic_on_overflow; -- cgit v1.2.3 From 802c1f6648aeb3eea670b4ef8b10014169b65699 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:34 -0300 Subject: x86: move dma_supported and dma_set_mask to pci-dma_32.c This is the way x86_64 does, so this make them equal. They have to be extern now in the header, and the extern definition is moved to the common dma-mapping.h header. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-dma_32.c | 33 +++++++++++++++++++++++++++++++++ include/asm-x86/dma-mapping.h | 3 +++ include/asm-x86/dma-mapping_32.h | 29 ----------------------------- include/asm-x86/dma-mapping_64.h | 4 ---- 4 files changed, 36 insertions(+), 33 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-dma_32.c b/arch/x86/kernel/pci-dma_32.c index 51330321a5d3..453b4bda2714 100644 --- a/arch/x86/kernel/pci-dma_32.c +++ b/arch/x86/kernel/pci-dma_32.c @@ -156,6 +156,39 @@ EXPORT_SYMBOL(dma_mark_declared_memory_occupied); int forbid_dac; EXPORT_SYMBOL(forbid_dac); +int +dma_supported(struct device *dev, u64 mask) +{ + /* + * we fall back to GFP_DMA when the mask isn't all 1s, + * so we can't guarantee allocations that must be + * within a tighter range than GFP_DMA.. + */ + if (mask < 0x00ffffff) + return 0; + + /* Work around chipset bugs */ + if (forbid_dac > 0 && mask > 0xffffffffULL) + return 0; + + if (dma_ops->dma_supported) + return dma_ops->dma_supported(dev, mask); + + return 1; +} + +int +dma_set_mask(struct device *dev, u64 mask) +{ + if (!dev->dma_mask || !dma_supported(dev, mask)) + return -EIO; + + *dev->dma_mask = mask; + + return 0; +} + + static __devinit void via_no_dac(struct pci_dev *dev) { if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 51a79d71dabd..c671a0aea59b 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -62,6 +62,9 @@ void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); +extern int dma_supported(struct device *hwdev, u64 mask); +extern int dma_set_mask(struct device *dev, u64 mask); + #ifdef CONFIG_X86_32 # include "dma-mapping_32.h" #else diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index e60c30a88311..fd7246dddad4 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -16,35 +16,6 @@ dma_mapping_error(dma_addr_t dma_addr) extern int forbid_dac; -static inline int -dma_supported(struct device *dev, u64 mask) -{ - /* - * we fall back to GFP_DMA when the mask isn't all 1s, - * so we can't guarantee allocations that must be - * within a tighter range than GFP_DMA.. - */ - if(mask < 0x00ffffff) - return 0; - - /* Work around chipset bugs */ - if (forbid_dac > 0 && mask > 0xffffffffULL) - return 0; - - return 1; -} - -static inline int -dma_set_mask(struct device *dev, u64 mask) -{ - if(!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - - return 0; -} - static inline int dma_get_cache_alignment(void) { diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index b1bc6ca7613d..9674dac9fa3a 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -12,8 +12,6 @@ static inline int dma_mapping_error(dma_addr_t dma_addr) return (dma_addr == bad_dma_address); } -extern int dma_supported(struct device *hwdev, u64 mask); - /* same for gart, swiotlb, and nommu */ static inline int dma_get_cache_alignment(void) { @@ -22,8 +20,6 @@ static inline int dma_get_cache_alignment(void) #define dma_is_consistent(d, h) 1 -extern int dma_set_mask(struct device *dev, u64 mask); - extern struct device fallback_dev; extern int panic_on_overflow; -- cgit v1.2.3 From b48ee7135230ac43b6820d59a784ac0bd51ae552 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:35 -0300 Subject: x86: align to clflush size Do it instead of using the conservative approach we're currently doing. This is the way x86_64 does, and this patch makes this piece of code the same between them, ready to be integrated. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping_32.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index fd7246dddad4..d0512c9251b7 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -21,7 +21,7 @@ dma_get_cache_alignment(void) { /* no easy way to get cache size on all x86, so return the * maximum possible, to be safe */ - return (1 << INTERNODE_CACHE_SHIFT); + return boot_cpu_data.x86_clflush_size; } #define dma_is_consistent(d, h) (1) -- cgit v1.2.3 From 7c18341665917b493fa40eeb3c7ff6c1a5ac47db Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:36 -0300 Subject: x86: provide a bad_dma_address symbol for i386 It's initially 0, since we don't expect any DMA there. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-dma_32.c | 4 ++++ include/asm-x86/dma-mapping.h | 2 ++ include/asm-x86/dma-mapping_64.h | 1 - 3 files changed, 6 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-dma_32.c b/arch/x86/kernel/pci-dma_32.c index 453b4bda2714..55ab3c874d8f 100644 --- a/arch/x86/kernel/pci-dma_32.c +++ b/arch/x86/kernel/pci-dma_32.c @@ -14,6 +14,10 @@ #include #include +/* For i386, we make it point to the NULL address */ +dma_addr_t bad_dma_address __read_mostly = 0x0; +EXPORT_SYMBOL(bad_dma_address); + struct dma_coherent_mem { void *virt_base; u32 device_base; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index c671a0aea59b..984935d86bbd 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -10,6 +10,8 @@ #include #include +extern dma_addr_t bad_dma_address; + struct dma_mapping_ops { int (*mapping_error)(dma_addr_t dma_addr); void* (*alloc_coherent)(struct device *dev, size_t size, diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 9674dac9fa3a..352bf4164a5e 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -1,7 +1,6 @@ #ifndef _X8664_DMA_MAPPING_H #define _X8664_DMA_MAPPING_H 1 -extern dma_addr_t bad_dma_address; extern int iommu_merge; static inline int dma_mapping_error(dma_addr_t dma_addr) -- cgit v1.2.3 From c786df08f6df2833e34e78cee5ef62558e3b5346 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:37 -0300 Subject: x86: unify dma_mapping_error We provide a map_error function in pci-base_32.c to make sure i386 keeps with the same behaviour it used to. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 7 +++++++ include/asm-x86/dma-mapping.h | 8 ++++++++ include/asm-x86/dma-mapping_32.h | 6 ------ include/asm-x86/dma-mapping_64.h | 8 -------- 4 files changed, 15 insertions(+), 14 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index cf4bb28dfc6a..7caf5c211f23 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -30,6 +30,12 @@ static int pci32_dma_map_sg(struct device *dev, struct scatterlist *sglist, return nents; } +/* Make sure we keep the same behaviour */ +static int pci32_map_error(dma_addr_t dma_addr) +{ + return 0; +} + static const struct dma_mapping_ops pci32_dma_ops = { .map_single = pci32_map_single, .unmap_single = NULL, @@ -41,6 +47,7 @@ static const struct dma_mapping_ops pci32_dma_ops = { .sync_single_range_for_device = NULL, .sync_sg_for_cpu = NULL, .sync_sg_for_device = NULL, + .mapping_error = pci32_map_error, }; const struct dma_mapping_ops *dma_ops = &pci32_dma_ops; diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 984935d86bbd..a7090bbb2a2d 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -54,6 +54,14 @@ struct dma_mapping_ops { extern const struct dma_mapping_ops *dma_ops; +static inline int dma_mapping_error(dma_addr_t dma_addr) +{ + if (dma_ops->mapping_error) + return dma_ops->mapping_error(dma_addr); + + return (dma_addr == bad_dma_address); +} + #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index d0512c9251b7..03a75f866534 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -8,12 +8,6 @@ #include #include -static inline int -dma_mapping_error(dma_addr_t dma_addr) -{ - return 0; -} - extern int forbid_dac; static inline int diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h index 352bf4164a5e..8353025f052d 100644 --- a/include/asm-x86/dma-mapping_64.h +++ b/include/asm-x86/dma-mapping_64.h @@ -3,14 +3,6 @@ extern int iommu_merge; -static inline int dma_mapping_error(dma_addr_t dma_addr) -{ - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dma_addr); - - return (dma_addr == bad_dma_address); -} - /* same for gart, swiotlb, and nommu */ static inline int dma_get_cache_alignment(void) { -- cgit v1.2.3 From ae17a63b096b05007bacafd2f92414b881a0b4b4 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:38 -0300 Subject: x86: move ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY to dma-mapping.h define it conditionally to i386. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 14 ++++++++++++++ include/asm-x86/dma-mapping_32.h | 12 ------------ 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index a7090bbb2a2d..1c88ce6b50d6 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -204,4 +204,18 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size, { flush_write_buffers(); } + +#ifdef CONFIG_X86_32 +# define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY +extern int +dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags); + +extern void +dma_release_declared_memory(struct device *dev); + +extern void * +dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size); +#endif /* CONFIG_X86_32 */ #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h index 03a75f866534..4ec4f45446cf 100644 --- a/include/asm-x86/dma-mapping_32.h +++ b/include/asm-x86/dma-mapping_32.h @@ -20,16 +20,4 @@ dma_get_cache_alignment(void) #define dma_is_consistent(d, h) (1) -#define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY -extern int -dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, - dma_addr_t device_addr, size_t size, int flags); - -extern void -dma_release_declared_memory(struct device *dev); - -extern void * -dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size); - #endif -- cgit v1.2.3 From b7107a3d9da2e122fb7f33dd1482254ff40fdf96 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 25 Mar 2008 18:36:39 -0300 Subject: x86: delete the arch-specific dma-mapping headers. all the code that is left is ready to be merged as-is in dma-mapping.h. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/dma-mapping.h | 19 +++++++++++++------ include/asm-x86/dma-mapping_32.h | 23 ----------------------- include/asm-x86/dma-mapping_64.h | 17 ----------------- 3 files changed, 13 insertions(+), 46 deletions(-) delete mode 100644 include/asm-x86/dma-mapping_32.h delete mode 100644 include/asm-x86/dma-mapping_64.h (limited to 'include') diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 1c88ce6b50d6..914846d0beaa 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -11,6 +11,9 @@ #include extern dma_addr_t bad_dma_address; +extern int iommu_merge; +extern struct device fallback_dev; +extern int panic_on_overflow; struct dma_mapping_ops { int (*mapping_error)(dma_addr_t dma_addr); @@ -75,12 +78,6 @@ void dma_free_coherent(struct device *dev, size_t size, extern int dma_supported(struct device *hwdev, u64 mask); extern int dma_set_mask(struct device *dev, u64 mask); -#ifdef CONFIG_X86_32 -# include "dma-mapping_32.h" -#else -# include "dma-mapping_64.h" -#endif - static inline dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size, int direction) @@ -205,6 +202,15 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size, flush_write_buffers(); } +static inline int dma_get_cache_alignment(void) +{ + /* no easy way to get cache size on all x86, so return the + * maximum possible, to be safe */ + return boot_cpu_data.x86_clflush_size; +} + +#define dma_is_consistent(d, h) (1) + #ifdef CONFIG_X86_32 # define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY extern int @@ -217,5 +223,6 @@ dma_release_declared_memory(struct device *dev); extern void * dma_mark_declared_memory_occupied(struct device *dev, dma_addr_t device_addr, size_t size); +extern int forbid_dac; #endif /* CONFIG_X86_32 */ #endif diff --git a/include/asm-x86/dma-mapping_32.h b/include/asm-x86/dma-mapping_32.h deleted file mode 100644 index 4ec4f45446cf..000000000000 --- a/include/asm-x86/dma-mapping_32.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _ASM_I386_DMA_MAPPING_H -#define _ASM_I386_DMA_MAPPING_H - -#include -#include - -#include -#include -#include - -extern int forbid_dac; - -static inline int -dma_get_cache_alignment(void) -{ - /* no easy way to get cache size on all x86, so return the - * maximum possible, to be safe */ - return boot_cpu_data.x86_clflush_size; -} - -#define dma_is_consistent(d, h) (1) - -#endif diff --git a/include/asm-x86/dma-mapping_64.h b/include/asm-x86/dma-mapping_64.h deleted file mode 100644 index 8353025f052d..000000000000 --- a/include/asm-x86/dma-mapping_64.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _X8664_DMA_MAPPING_H -#define _X8664_DMA_MAPPING_H 1 - -extern int iommu_merge; - -/* same for gart, swiotlb, and nommu */ -static inline int dma_get_cache_alignment(void) -{ - return boot_cpu_data.x86_clflush_size; -} - -#define dma_is_consistent(d, h) 1 - -extern struct device fallback_dev; -extern int panic_on_overflow; - -#endif /* _X8664_DMA_MAPPING_H */ -- cgit v1.2.3 From d741bde26dc3444eaeb269051d3f0b623b24de13 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 8 Apr 2008 13:20:48 -0300 Subject: x86: use dma_length in i386 This is done to get the code closer to x86_64. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-base_32.c | 1 + include/asm-x86/scatterlist.h | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-base_32.c b/arch/x86/kernel/pci-base_32.c index 7caf5c211f23..837bbe91043e 100644 --- a/arch/x86/kernel/pci-base_32.c +++ b/arch/x86/kernel/pci-base_32.c @@ -24,6 +24,7 @@ static int pci32_dma_map_sg(struct device *dev, struct scatterlist *sglist, BUG_ON(!sg_page(sg)); sg->dma_address = sg_phys(sg); + sg->dma_length = sg->length; } flush_write_buffers(); diff --git a/include/asm-x86/scatterlist.h b/include/asm-x86/scatterlist.h index d13c197866d6..c0432061f81a 100644 --- a/include/asm-x86/scatterlist.h +++ b/include/asm-x86/scatterlist.h @@ -11,9 +11,7 @@ struct scatterlist { unsigned int offset; unsigned int length; dma_addr_t dma_address; -#ifdef CONFIG_X86_64 unsigned int dma_length; -#endif }; #define ARCH_HAS_SG_CHAIN -- cgit v1.2.3 From bca5c09663030bdd18ab1b3ccb6671f663c3345a Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 8 Apr 2008 13:20:53 -0300 Subject: x86: move pci fixup to pci-dma.c via_no_dac provides a fixup that is the same for both architectures. Move it to pci-dma.c. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-dma.c | 18 ++++++++++++++++++ arch/x86/kernel/pci-dma_32.c | 13 ------------- arch/x86/kernel/pci-dma_64.c | 15 --------------- include/asm-x86/dma-mapping.h | 2 +- 4 files changed, 19 insertions(+), 29 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 91443361cb67..48cccbe51aa5 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -1,12 +1,16 @@ #include #include #include +#include #include #include #include #include +int forbid_dac __read_mostly; +EXPORT_SYMBOL(forbid_dac); + const struct dma_mapping_ops *dma_ops; EXPORT_SYMBOL(dma_ops); @@ -121,3 +125,17 @@ void pci_iommu_shutdown(void) } /* Must execute after PCI subsystem */ fs_initcall(pci_iommu_init); + +#ifdef CONFIG_PCI +/* Many VIA bridges seem to corrupt data for DAC. Disable it here */ + +static __devinit void via_no_dac(struct pci_dev *dev) +{ + if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { + printk(KERN_INFO "PCI: VIA PCI bridge detected." + "Disabling DAC.\n"); + forbid_dac = 1; + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); +#endif diff --git a/arch/x86/kernel/pci-dma_32.c b/arch/x86/kernel/pci-dma_32.c index 9e8297657c32..6543bb30b65d 100644 --- a/arch/x86/kernel/pci-dma_32.c +++ b/arch/x86/kernel/pci-dma_32.c @@ -157,9 +157,6 @@ EXPORT_SYMBOL(dma_mark_declared_memory_occupied); #ifdef CONFIG_PCI /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ -int forbid_dac; -EXPORT_SYMBOL(forbid_dac); - int dma_supported(struct device *dev, u64 mask) { @@ -182,16 +179,6 @@ dma_supported(struct device *dev, u64 mask) } EXPORT_SYMBOL(dma_supported); - -static __devinit void via_no_dac(struct pci_dev *dev) -{ - if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { - printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n"); - forbid_dac = 1; - } -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); - static int check_iommu(char *s) { if (!strcmp(s, "usedac")) { diff --git a/arch/x86/kernel/pci-dma_64.c b/arch/x86/kernel/pci-dma_64.c index 6b204cc42890..7820675a688a 100644 --- a/arch/x86/kernel/pci-dma_64.c +++ b/arch/x86/kernel/pci-dma_64.c @@ -161,8 +161,6 @@ void dma_free_coherent(struct device *dev, size_t size, } EXPORT_SYMBOL(dma_free_coherent); -static int forbid_dac __read_mostly; - int dma_supported(struct device *dev, u64 mask) { #ifdef CONFIG_PCI @@ -270,16 +268,3 @@ static __init int iommu_setup(char *p) return 0; } early_param("iommu", iommu_setup); - -#ifdef CONFIG_PCI -/* Many VIA bridges seem to corrupt data for DAC. Disable it here */ - -static __devinit void via_no_dac(struct pci_dev *dev) -{ - if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { - printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n"); - forbid_dac = 1; - } -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); -#endif diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 914846d0beaa..d82517de1e7c 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -14,6 +14,7 @@ extern dma_addr_t bad_dma_address; extern int iommu_merge; extern struct device fallback_dev; extern int panic_on_overflow; +extern int forbid_dac; struct dma_mapping_ops { int (*mapping_error)(dma_addr_t dma_addr); @@ -223,6 +224,5 @@ dma_release_declared_memory(struct device *dev); extern void * dma_mark_declared_memory_occupied(struct device *dev, dma_addr_t device_addr, size_t size); -extern int forbid_dac; #endif /* CONFIG_X86_32 */ #endif -- cgit v1.2.3 From fae9a0d8ca68a14da8d2351ad3e0bf42f3b29899 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 8 Apr 2008 13:20:56 -0300 Subject: x86: merge iommu initialization parameters we merge the iommu initialization parameters in pci-dma.c Nice thing, that both architectures at least recognize the same parameters. usedac i386 parameter is marked for deprecation Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- Documentation/feature-removal-schedule.txt | 7 +++ arch/x86/kernel/pci-dma.c | 81 ++++++++++++++++++++++++++++++ arch/x86/kernel/pci-dma_32.c | 12 ----- arch/x86/kernel/pci-dma_64.c | 79 ----------------------------- include/asm-x86/dma-mapping.h | 1 + 5 files changed, 89 insertions(+), 91 deletions(-) (limited to 'include') diff --git a/Documentation/feature-removal-schedule.txt b/Documentation/feature-removal-schedule.txt index af0e9393bf68..309c47b91598 100644 --- a/Documentation/feature-removal-schedule.txt +++ b/Documentation/feature-removal-schedule.txt @@ -282,6 +282,13 @@ Why: Not used in-tree. The current out-of-tree users used it to out-of-tree driver. Who: Thomas Gleixner +---------------------------- + +What: usedac i386 kernel parameter +When: 2.6.27 +Why: replaced by allowdac and no dac combination +Who: Glauber Costa + --------------------------- What: /sys/o2cb symlink diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 7d3bd652c36f..48ab52d052b6 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -24,6 +24,18 @@ int panic_on_overflow __read_mostly = 0; int force_iommu __read_mostly = 0; #endif +int iommu_merge __read_mostly = 0; + +int no_iommu __read_mostly; +/* Set this to 1 if there is a HW IOMMU in the system */ +int iommu_detected __read_mostly = 0; + +/* This tells the BIO block layer to assume merging. Default to off + because we cannot guarantee merging later. */ +int iommu_bio_merge __read_mostly = 0; +EXPORT_SYMBOL(iommu_bio_merge); + + int dma_set_mask(struct device *dev, u64 mask) { if (!dev->dma_mask || !dma_supported(dev, mask)) @@ -105,6 +117,75 @@ void __init pci_iommu_alloc(void) } #endif +/* + * See for the iommu kernel parameter + * documentation. + */ +static __init int iommu_setup(char *p) +{ + iommu_merge = 1; + + if (!p) + return -EINVAL; + + while (*p) { + if (!strncmp(p, "off", 3)) + no_iommu = 1; + /* gart_parse_options has more force support */ + if (!strncmp(p, "force", 5)) + force_iommu = 1; + if (!strncmp(p, "noforce", 7)) { + iommu_merge = 0; + force_iommu = 0; + } + + if (!strncmp(p, "biomerge", 8)) { + iommu_bio_merge = 4096; + iommu_merge = 1; + force_iommu = 1; + } + if (!strncmp(p, "panic", 5)) + panic_on_overflow = 1; + if (!strncmp(p, "nopanic", 7)) + panic_on_overflow = 0; + if (!strncmp(p, "merge", 5)) { + iommu_merge = 1; + force_iommu = 1; + } + if (!strncmp(p, "nomerge", 7)) + iommu_merge = 0; + if (!strncmp(p, "forcesac", 8)) + iommu_sac_force = 1; + if (!strncmp(p, "allowdac", 8)) + forbid_dac = 0; + if (!strncmp(p, "nodac", 5)) + forbid_dac = -1; + if (!strncmp(p, "usedac", 6)) { + forbid_dac = -1; + return 1; + } +#ifdef CONFIG_SWIOTLB + if (!strncmp(p, "soft", 4)) + swiotlb = 1; +#endif + +#ifdef CONFIG_GART_IOMMU + gart_parse_options(p); +#endif + +#ifdef CONFIG_CALGARY_IOMMU + if (!strncmp(p, "calgary", 7)) + use_calgary = 1; +#endif /* CONFIG_CALGARY_IOMMU */ + + p += strcspn(p, ","); + if (*p == ',') + ++p; + } + return 0; +} +early_param("iommu", iommu_setup); + int dma_supported(struct device *dev, u64 mask) { #ifdef CONFIG_PCI diff --git a/arch/x86/kernel/pci-dma_32.c b/arch/x86/kernel/pci-dma_32.c index 1d4091af4417..eea52df68a3b 100644 --- a/arch/x86/kernel/pci-dma_32.c +++ b/arch/x86/kernel/pci-dma_32.c @@ -153,15 +153,3 @@ void *dma_mark_declared_memory_occupied(struct device *dev, return mem->virt_base + (pos << PAGE_SHIFT); } EXPORT_SYMBOL(dma_mark_declared_memory_occupied); - -#ifdef CONFIG_PCI -static int check_iommu(char *s) -{ - if (!strcmp(s, "usedac")) { - forbid_dac = -1; - return 1; - } - return 0; -} -__setup("iommu=", check_iommu); -#endif diff --git a/arch/x86/kernel/pci-dma_64.c b/arch/x86/kernel/pci-dma_64.c index c80da76e7e61..e7d45cf82251 100644 --- a/arch/x86/kernel/pci-dma_64.c +++ b/arch/x86/kernel/pci-dma_64.c @@ -14,22 +14,9 @@ #include #include -int iommu_merge __read_mostly = 0; - dma_addr_t bad_dma_address __read_mostly; EXPORT_SYMBOL(bad_dma_address); -/* This tells the BIO block layer to assume merging. Default to off - because we cannot guarantee merging later. */ -int iommu_bio_merge __read_mostly = 0; -EXPORT_SYMBOL(iommu_bio_merge); - -extern int iommu_sac_force; - -int no_iommu __read_mostly; -/* Set this to 1 if there is a HW IOMMU in the system */ -int iommu_detected __read_mostly = 0; - /* Dummy device used for NULL arguments (normally ISA). Better would be probably a smaller DMA mask, but this is bug-to-bug compatible to i386. */ @@ -160,69 +147,3 @@ void dma_free_coherent(struct device *dev, size_t size, free_pages((unsigned long)vaddr, get_order(size)); } EXPORT_SYMBOL(dma_free_coherent); - -/* - * See for the iommu kernel parameter - * documentation. - */ -static __init int iommu_setup(char *p) -{ - iommu_merge = 1; - - if (!p) - return -EINVAL; - - while (*p) { - if (!strncmp(p, "off", 3)) - no_iommu = 1; - /* gart_parse_options has more force support */ - if (!strncmp(p, "force", 5)) - force_iommu = 1; - if (!strncmp(p, "noforce", 7)) { - iommu_merge = 0; - force_iommu = 0; - } - - if (!strncmp(p, "biomerge", 8)) { - iommu_bio_merge = 4096; - iommu_merge = 1; - force_iommu = 1; - } - if (!strncmp(p, "panic", 5)) - panic_on_overflow = 1; - if (!strncmp(p, "nopanic", 7)) - panic_on_overflow = 0; - if (!strncmp(p, "merge", 5)) { - iommu_merge = 1; - force_iommu = 1; - } - if (!strncmp(p, "nomerge", 7)) - iommu_merge = 0; - if (!strncmp(p, "forcesac", 8)) - iommu_sac_force = 1; - if (!strncmp(p, "allowdac", 8)) - forbid_dac = 0; - if (!strncmp(p, "nodac", 5)) - forbid_dac = -1; - -#ifdef CONFIG_SWIOTLB - if (!strncmp(p, "soft", 4)) - swiotlb = 1; -#endif - -#ifdef CONFIG_GART_IOMMU - gart_parse_options(p); -#endif - -#ifdef CONFIG_CALGARY_IOMMU - if (!strncmp(p, "calgary", 7)) - use_calgary = 1; -#endif /* CONFIG_CALGARY_IOMMU */ - - p += strcspn(p, ","); - if (*p == ',') - ++p; - } - return 0; -} -early_param("iommu", iommu_setup); diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index d82517de1e7c..75807368051a 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -15,6 +15,7 @@ extern int iommu_merge; extern struct device fallback_dev; extern int panic_on_overflow; extern int forbid_dac; +extern int force_iommu; struct dma_mapping_ops { int (*mapping_error)(dma_addr_t dma_addr); -- cgit v1.2.3 From 8e8edc6401205da3000cc3dfa76f3fd28a21d73c Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 8 Apr 2008 13:20:57 -0300 Subject: x86: move dma_coherent functions to pci-dma.c They are placed in an ifdef, since they are i386 specific the structure definition goes to dma-mapping.h. Signed-off-by: Glauber Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/pci-dma.c | 81 +++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/pci-dma_32.c | 85 ------------------------------------------- include/asm-x86/dma-mapping.h | 8 ++++ 3 files changed, 89 insertions(+), 85 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 48ab52d052b6..967dfcfa2ad2 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -186,6 +186,87 @@ static __init int iommu_setup(char *p) } early_param("iommu", iommu_setup); +#ifdef CONFIG_X86_32 +int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags) +{ + void __iomem *mem_base = NULL; + int pages = size >> PAGE_SHIFT; + int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + + if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) + goto out; + if (!size) + goto out; + if (dev->dma_mem) + goto out; + + /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ + + mem_base = ioremap(bus_addr, size); + if (!mem_base) + goto out; + + dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); + if (!dev->dma_mem) + goto out; + dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); + if (!dev->dma_mem->bitmap) + goto free1_out; + + dev->dma_mem->virt_base = mem_base; + dev->dma_mem->device_base = device_addr; + dev->dma_mem->size = pages; + dev->dma_mem->flags = flags; + + if (flags & DMA_MEMORY_MAP) + return DMA_MEMORY_MAP; + + return DMA_MEMORY_IO; + + free1_out: + kfree(dev->dma_mem); + out: + if (mem_base) + iounmap(mem_base); + return 0; +} +EXPORT_SYMBOL(dma_declare_coherent_memory); + +void dma_release_declared_memory(struct device *dev) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + + if (!mem) + return; + dev->dma_mem = NULL; + iounmap(mem->virt_base); + kfree(mem->bitmap); + kfree(mem); +} +EXPORT_SYMBOL(dma_release_declared_memory); + +void *dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + int pos, err; + int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1); + + pages >>= PAGE_SHIFT; + + if (!mem) + return ERR_PTR(-EINVAL); + + pos = (device_addr - mem->device_base) >> PAGE_SHIFT; + err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages)); + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); +} +EXPORT_SYMBOL(dma_mark_declared_memory_occupied); +#endif /* CONFIG_X86_32 */ + int dma_supported(struct device *dev, u64 mask) { #ifdef CONFIG_PCI diff --git a/arch/x86/kernel/pci-dma_32.c b/arch/x86/kernel/pci-dma_32.c index eea52df68a3b..818d95efc3cb 100644 --- a/arch/x86/kernel/pci-dma_32.c +++ b/arch/x86/kernel/pci-dma_32.c @@ -18,14 +18,6 @@ dma_addr_t bad_dma_address __read_mostly = 0x0; EXPORT_SYMBOL(bad_dma_address); -struct dma_coherent_mem { - void *virt_base; - u32 device_base; - int size; - int flags; - unsigned long *bitmap; -}; - void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp) { @@ -76,80 +68,3 @@ void dma_free_coherent(struct device *dev, size_t size, free_pages((unsigned long)vaddr, order); } EXPORT_SYMBOL(dma_free_coherent); - -int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, - dma_addr_t device_addr, size_t size, int flags) -{ - void __iomem *mem_base = NULL; - int pages = size >> PAGE_SHIFT; - int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); - - if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) - goto out; - if (!size) - goto out; - if (dev->dma_mem) - goto out; - - /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ - - mem_base = ioremap(bus_addr, size); - if (!mem_base) - goto out; - - dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); - if (!dev->dma_mem) - goto out; - dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); - if (!dev->dma_mem->bitmap) - goto free1_out; - - dev->dma_mem->virt_base = mem_base; - dev->dma_mem->device_base = device_addr; - dev->dma_mem->size = pages; - dev->dma_mem->flags = flags; - - if (flags & DMA_MEMORY_MAP) - return DMA_MEMORY_MAP; - - return DMA_MEMORY_IO; - - free1_out: - kfree(dev->dma_mem); - out: - if (mem_base) - iounmap(mem_base); - return 0; -} -EXPORT_SYMBOL(dma_declare_coherent_memory); - -void dma_release_declared_memory(struct device *dev) -{ - struct dma_coherent_mem *mem = dev->dma_mem; - - if(!mem) - return; - dev->dma_mem = NULL; - iounmap(mem->virt_base); - kfree(mem->bitmap); - kfree(mem); -} -EXPORT_SYMBOL(dma_release_declared_memory); - -void *dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size) -{ - struct dma_coherent_mem *mem = dev->dma_mem; - int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT; - int pos, err; - - if (!mem) - return ERR_PTR(-EINVAL); - - pos = (device_addr - mem->device_base) >> PAGE_SHIFT; - err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages)); - if (err != 0) - return ERR_PTR(err); - return mem->virt_base + (pos << PAGE_SHIFT); -} -EXPORT_SYMBOL(dma_mark_declared_memory_occupied); diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h index 75807368051a..a1a4dc7fe6ec 100644 --- a/include/asm-x86/dma-mapping.h +++ b/include/asm-x86/dma-mapping.h @@ -215,6 +215,14 @@ static inline int dma_get_cache_alignment(void) #ifdef CONFIG_X86_32 # define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY +struct dma_coherent_mem { + void *virt_base; + u32 device_base; + int size; + int flags; + unsigned long *bitmap; +}; + extern int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, dma_addr_t device_addr, size_t size, int flags); -- cgit v1.2.3 From 34d0559178393547505ec9492321255405f4e441 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Wed, 16 Apr 2008 11:45:15 -0500 Subject: x86: UV startup of slave cpus This patch changes smpboot.c so that it can start slave cpus running in UV non-unique apicid mode. The SIPI must be sent using a UV-specific mechanism. Signed-off-by: Jack Steiner Signed-off-by: Ingo Molnar --- arch/x86/kernel/genx2apic_uv_x.c | 17 +++++++++++------ arch/x86/kernel/smpboot.c | 29 ++++++++++++++++++++--------- include/asm-x86/genapic_32.h | 1 + 3 files changed, 32 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/arch/x86/kernel/genx2apic_uv_x.c b/arch/x86/kernel/genx2apic_uv_x.c index 5d77c9cd8e15..ebf13908a743 100644 --- a/arch/x86/kernel/genx2apic_uv_x.c +++ b/arch/x86/kernel/genx2apic_uv_x.c @@ -61,26 +61,31 @@ int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip) val = (1UL << UVH_IPI_INT_SEND_SHFT) | (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) | (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) | - (6 << UVH_IPI_INT_DELIVERY_MODE_SHFT); + APIC_DM_INIT; + uv_write_global_mmr64(nasid, UVH_IPI_INT, val); + mdelay(10); + + val = (1UL << UVH_IPI_INT_SEND_SHFT) | + (phys_apicid << UVH_IPI_INT_APIC_ID_SHFT) | + (((long)start_rip << UVH_IPI_INT_VECTOR_SHFT) >> 12) | + APIC_DM_STARTUP; uv_write_global_mmr64(nasid, UVH_IPI_INT, val); return 0; } static void uv_send_IPI_one(int cpu, int vector) { - unsigned long val, apicid; + unsigned long val, apicid, lapicid; int nasid; apicid = per_cpu(x86_cpu_to_apicid, cpu); /* ZZZ - cache node-local ? */ + lapicid = apicid & 0x3f; /* ZZZ macro needed */ nasid = uv_apicid_to_nasid(apicid); val = - (1UL << UVH_IPI_INT_SEND_SHFT) | (apicid << + (1UL << UVH_IPI_INT_SEND_SHFT) | (lapicid << UVH_IPI_INT_APIC_ID_SHFT) | (vector << UVH_IPI_INT_VECTOR_SHFT); uv_write_global_mmr64(nasid, UVH_IPI_INT, val); - printk(KERN_DEBUG - "UV: IPI to cpu %d, apicid 0x%lx, vec %d, nasid%d, val 0x%lx\n", - cpu, apicid, vector, nasid, val); } static void uv_send_IPI_mask(cpumask_t mask, int vector) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index e6abe8a49b1f..6a925394bc7e 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -61,6 +61,7 @@ #include #include #include +#include #include #include @@ -677,6 +678,12 @@ wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip) unsigned long send_status, accept_status = 0; int maxlvt, num_starts, j; + if (get_uv_system_type() == UV_NON_UNIQUE_APIC) { + send_status = uv_wakeup_secondary(phys_apicid, start_eip); + atomic_set(&init_deasserted, 1); + return send_status; + } + /* * Be paranoid about clearing APIC errors. */ @@ -918,16 +925,19 @@ do_rest: atomic_set(&init_deasserted, 0); - Dprintk("Setting warm reset code and vector.\n"); + if (get_uv_system_type() != UV_NON_UNIQUE_APIC) { - store_NMI_vector(&nmi_high, &nmi_low); + Dprintk("Setting warm reset code and vector.\n"); - smpboot_setup_warm_reset_vector(start_ip); - /* - * Be paranoid about clearing APIC errors. - */ - apic_write(APIC_ESR, 0); - apic_read(APIC_ESR); + store_NMI_vector(&nmi_high, &nmi_low); + + smpboot_setup_warm_reset_vector(start_ip); + /* + * Be paranoid about clearing APIC errors. + */ + apic_write(APIC_ESR, 0); + apic_read(APIC_ESR); + } /* * Starting actual IPI sequence... @@ -966,7 +976,8 @@ do_rest: else /* trampoline code not run */ printk(KERN_ERR "Not responding.\n"); - inquire_remote_apic(apicid); + if (get_uv_system_type() != UV_NON_UNIQUE_APIC) + inquire_remote_apic(apicid); } } diff --git a/include/asm-x86/genapic_32.h b/include/asm-x86/genapic_32.h index f1b96932746b..b02ea6e17de8 100644 --- a/include/asm-x86/genapic_32.h +++ b/include/asm-x86/genapic_32.h @@ -117,6 +117,7 @@ extern struct genapic *genapic; enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC}; #define get_uv_system_type() UV_NONE #define is_uv_system() 0 +#define uv_wakeup_secondary(a, b) 1 #endif -- cgit v1.2.3