sparc: Use popc when possible for ffs/__ffs/ffz.

Signed-off-by: David S. Miller <davem@davemloft.net>

4 files changed, 37 insertions, 11 deletions

diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h
index 16d1545d84fc..e27f8ea8656e 100644
--- a/arch/sparc/kernel/entry.h
+++ b/arch/sparc/kernel/entry.h
@@ -49,6 +49,13 @@ struct popc_3insn_patch_entry {
 extern struct popc_3insn_patch_entry __popc_3insn_patch,
 	__popc_3insn_patch_end;
 
+struct popc_6insn_patch_entry {
+	unsigned int	addr;
+	unsigned int	insns[6];
+};
+extern struct popc_6insn_patch_entry __popc_6insn_patch,
+	__popc_6insn_patch_end;
+
 extern void __init per_cpu_patch(void);
 extern void __init sun4v_patch(void);
 extern void __init boot_cpu_id_too_large(int cpu);
diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c
index 26d114187e10..3e9daea1653d 100644
--- a/arch/sparc/kernel/setup_64.c
+++ b/arch/sparc/kernel/setup_64.c
@@ -275,24 +275,34 @@ void __init sun4v_patch(void)
 static void __init popc_patch(void)
 {
 	struct popc_3insn_patch_entry *p3;
+	struct popc_6insn_patch_entry *p6;
 
 	p3 = &__popc_3insn_patch;
 	while (p3 < &__popc_3insn_patch_end) {
-		unsigned long addr = p3->addr;
+		unsigned long i, addr = p3->addr;
 
-		*(unsigned int *) (addr +  0) = p3->insns[0];
-		wmb();
-		__asm__ __volatile__("flush	%0" : : "r" (addr +  0));
+		for (i = 0; i < 3; i++) {
+			*(unsigned int *) (addr +  (i * 4)) = p3->insns[i];
+			wmb();
+			__asm__ __volatile__("flush	%0"
+					     : : "r" (addr +  (i * 4)));
+		}
 
-		*(unsigned int *) (addr +  4) = p3->insns[1];
-		wmb();
-		__asm__ __volatile__("flush	%0" : : "r" (addr +  4));
+		p3++;
+	}
 
-		*(unsigned int *) (addr +  8) = p3->insns[2];
-		wmb();
-		__asm__ __volatile__("flush	%0" : : "r" (addr +  4));
+	p6 = &__popc_6insn_patch;
+	while (p6 < &__popc_6insn_patch_end) {
+		unsigned long i, addr = p6->addr;
 
-		p3++;
+		for (i = 0; i < 6; i++) {
+			*(unsigned int *) (addr +  (i * 4)) = p6->insns[i];
+			wmb();
+			__asm__ __volatile__("flush	%0"
+					     : : "r" (addr +  (i * 4)));
+		}
+
+		p6++;
 	}
 }
 
diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
index d0ee65aced0d..83b47ab02d96 100644
--- a/arch/sparc/kernel/sparc_ksyms_64.c
+++ b/arch/sparc/kernel/sparc_ksyms_64.c
@@ -45,5 +45,9 @@ EXPORT_SYMBOL(__arch_hweight16);
 EXPORT_SYMBOL(__arch_hweight32);
 EXPORT_SYMBOL(__arch_hweight64);
 
+/* from ffs_ffz.S */
+EXPORT_SYMBOL(ffs);
+EXPORT_SYMBOL(__ffs);
+
 /* Exporting a symbol from /init/main.c */
 EXPORT_SYMBOL(saved_command_line);
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index de20c14625eb..94a954892d3f 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -112,6 +112,11 @@ SECTIONS
 		*(.popc_3insn_patch)
 		__popc_3insn_patch_end = .;
 	}
+	.popc_6insn_patch : {
+		__popc_6insn_patch = .;
+		*(.popc_6insn_patch)
+		__popc_6insn_patch_end = .;
+	}
 	PERCPU_SECTION(SMP_CACHE_BYTES)
 
 	. = ALIGN(PAGE_SIZE);