summaryrefslogtreecommitdiffstats
path: root/arch/x86/crypto/nh-sse2-x86_64.S
blob: 75fb994b6d177eb3f4f8fdea39182aec2cca6e1e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
 *
 * Copyright 2018 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>
#include <linux/cfi_types.h>

#define		PASS0_SUMS	%xmm0
#define		PASS1_SUMS	%xmm1
#define		PASS2_SUMS	%xmm2
#define		PASS3_SUMS	%xmm3
#define		K0		%xmm4
#define		K1		%xmm5
#define		K2		%xmm6
#define		K3		%xmm7
#define		T0		%xmm8
#define		T1		%xmm9
#define		T2		%xmm10
#define		T3		%xmm11
#define		T4		%xmm12
#define		T5		%xmm13
#define		T6		%xmm14
#define		T7		%xmm15
#define		KEY		%rdi
#define		MESSAGE		%rsi
#define		MESSAGE_LEN	%rdx
#define		HASH		%rcx

.macro _nh_stride	k0, k1, k2, k3, offset

	// Load next message stride
	movdqu		\offset(MESSAGE), T1

	// Load next key stride
	movdqu		\offset(KEY), \k3

	// Add message words to key words
	movdqa		T1, T2
	movdqa		T1, T3
	paddd		T1, \k0    // reuse k0 to avoid a move
	paddd		\k1, T1
	paddd		\k2, T2
	paddd		\k3, T3

	// Multiply 32x32 => 64 and accumulate
	pshufd		$0x10, \k0, T4
	pshufd		$0x32, \k0, \k0
	pshufd		$0x10, T1, T5
	pshufd		$0x32, T1, T1
	pshufd		$0x10, T2, T6
	pshufd		$0x32, T2, T2
	pshufd		$0x10, T3, T7
	pshufd		$0x32, T3, T3
	pmuludq		T4, \k0
	pmuludq		T5, T1
	pmuludq		T6, T2
	pmuludq		T7, T3
	paddq		\k0, PASS0_SUMS
	paddq		T1, PASS1_SUMS
	paddq		T2, PASS2_SUMS
	paddq		T3, PASS3_SUMS
.endm

/*
 * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
 *		__le64 hash[NH_NUM_PASSES])
 *
 * It's guaranteed that message_len % 16 == 0.
 */
SYM_TYPED_FUNC_START(nh_sse2)

	movdqu		0x00(KEY), K0
	movdqu		0x10(KEY), K1
	movdqu		0x20(KEY), K2
	add		$0x30, KEY
	pxor		PASS0_SUMS, PASS0_SUMS
	pxor		PASS1_SUMS, PASS1_SUMS
	pxor		PASS2_SUMS, PASS2_SUMS
	pxor		PASS3_SUMS, PASS3_SUMS

	sub		$0x40, MESSAGE_LEN
	jl		.Lloop4_done
.Lloop4:
	_nh_stride	K0, K1, K2, K3, 0x00
	_nh_stride	K1, K2, K3, K0, 0x10
	_nh_stride	K2, K3, K0, K1, 0x20
	_nh_stride	K3, K0, K1, K2, 0x30
	add		$0x40, KEY
	add		$0x40, MESSAGE
	sub		$0x40, MESSAGE_LEN
	jge		.Lloop4

.Lloop4_done:
	and		$0x3f, MESSAGE_LEN
	jz		.Ldone
	_nh_stride	K0, K1, K2, K3, 0x00

	sub		$0x10, MESSAGE_LEN
	jz		.Ldone
	_nh_stride	K1, K2, K3, K0, 0x10

	sub		$0x10, MESSAGE_LEN
	jz		.Ldone
	_nh_stride	K2, K3, K0, K1, 0x20

.Ldone:
	// Sum the accumulators for each pass, then store the sums to 'hash'
	movdqa		PASS0_SUMS, T0
	movdqa		PASS2_SUMS, T1
	punpcklqdq	PASS1_SUMS, T0		// => (PASS0_SUM_A PASS1_SUM_A)
	punpcklqdq	PASS3_SUMS, T1		// => (PASS2_SUM_A PASS3_SUM_A)
	punpckhqdq	PASS1_SUMS, PASS0_SUMS	// => (PASS0_SUM_B PASS1_SUM_B)
	punpckhqdq	PASS3_SUMS, PASS2_SUMS	// => (PASS2_SUM_B PASS3_SUM_B)
	paddq		PASS0_SUMS, T0
	paddq		PASS2_SUMS, T1
	movdqu		T0, 0x00(HASH)
	movdqu		T1, 0x10(HASH)
	RET
SYM_FUNC_END(nh_sse2)