summaryrefslogtreecommitdiffstats
path: root/arch/arm/crypto/nh-neon-core.S
blob: 01620a0782ca93f2aa34730b526fe25f1c273e83 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * NH - ε-almost-universal hash function, NEON accelerated version
 *
 * Copyright 2018 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

	.text
	.fpu		neon

	KEY		.req	r0
	MESSAGE		.req	r1
	MESSAGE_LEN	.req	r2
	HASH		.req	r3

	PASS0_SUMS	.req	q0
	PASS0_SUM_A	.req	d0
	PASS0_SUM_B	.req	d1
	PASS1_SUMS	.req	q1
	PASS1_SUM_A	.req	d2
	PASS1_SUM_B	.req	d3
	PASS2_SUMS	.req	q2
	PASS2_SUM_A	.req	d4
	PASS2_SUM_B	.req	d5
	PASS3_SUMS	.req	q3
	PASS3_SUM_A	.req	d6
	PASS3_SUM_B	.req	d7
	K0		.req	q4
	K1		.req	q5
	K2		.req	q6
	K3		.req	q7
	T0		.req	q8
	T0_L		.req	d16
	T0_H		.req	d17
	T1		.req	q9
	T1_L		.req	d18
	T1_H		.req	d19
	T2		.req	q10
	T2_L		.req	d20
	T2_H		.req	d21
	T3		.req	q11
	T3_L		.req	d22
	T3_H		.req	d23

.macro _nh_stride	k0, k1, k2, k3

	// Load next message stride
	vld1.8		{T3}, [MESSAGE]!

	// Load next key stride
	vld1.32		{\k3}, [KEY]!

	// Add message words to key words
	vadd.u32	T0, T3, \k0
	vadd.u32	T1, T3, \k1
	vadd.u32	T2, T3, \k2
	vadd.u32	T3, T3, \k3

	// Multiply 32x32 => 64 and accumulate
	vmlal.u32	PASS0_SUMS, T0_L, T0_H
	vmlal.u32	PASS1_SUMS, T1_L, T1_H
	vmlal.u32	PASS2_SUMS, T2_L, T2_H
	vmlal.u32	PASS3_SUMS, T3_L, T3_H
.endm

/*
 * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
 *		__le64 hash[NH_NUM_PASSES])
 *
 * It's guaranteed that message_len % 16 == 0.
 */
ENTRY(nh_neon)

	vld1.32		{K0,K1}, [KEY]!
	  vmov.u64	PASS0_SUMS, #0
	  vmov.u64	PASS1_SUMS, #0
	vld1.32		{K2}, [KEY]!
	  vmov.u64	PASS2_SUMS, #0
	  vmov.u64	PASS3_SUMS, #0

	subs		MESSAGE_LEN, MESSAGE_LEN, #64
	blt		.Lloop4_done
.Lloop4:
	_nh_stride	K0, K1, K2, K3
	_nh_stride	K1, K2, K3, K0
	_nh_stride	K2, K3, K0, K1
	_nh_stride	K3, K0, K1, K2
	subs		MESSAGE_LEN, MESSAGE_LEN, #64
	bge		.Lloop4

.Lloop4_done:
	ands		MESSAGE_LEN, MESSAGE_LEN, #63
	beq		.Ldone
	_nh_stride	K0, K1, K2, K3

	subs		MESSAGE_LEN, MESSAGE_LEN, #16
	beq		.Ldone
	_nh_stride	K1, K2, K3, K0

	subs		MESSAGE_LEN, MESSAGE_LEN, #16
	beq		.Ldone
	_nh_stride	K2, K3, K0, K1

.Ldone:
	// Sum the accumulators for each pass, then store the sums to 'hash'
	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
	vst1.8		{T0-T1}, [HASH]
	bx		lr
ENDPROC(nh_neon)