Text file src/crypto/sha256/sha256block_amd64.s
1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "textflag.h"
6
7 // SHA256 block routine. See sha256block.go for Go equivalent.
8 //
9 // The algorithm is detailed in FIPS 180-4:
10 //
11 // https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf
12
13 // The avx2-version is described in an Intel White-Paper:
14 // "Fast SHA-256 Implementations on Intel Architecture Processors"
15 // To find it, surf to http://www.intel.com/p/en_US/embedded
16 // and search for that title.
17 // AVX2 version by Intel, same algorithm as code in Linux kernel:
18 // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha256-avx2-asm.S
19 // by
20 // James Guilford <james.guilford@intel.com>
21 // Kirk Yap <kirk.s.yap@intel.com>
22 // Tim Chen <tim.c.chen@linux.intel.com>
23
24 // Wt = Mt; for 0 <= t <= 15
25 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63
26 //
27 // a = H0
28 // b = H1
29 // c = H2
30 // d = H3
31 // e = H4
32 // f = H5
33 // g = H6
34 // h = H7
35 //
36 // for t = 0 to 63 {
37 // T1 = h + BIGSIGMA1(e) + Ch(e,f,g) + Kt + Wt
38 // T2 = BIGSIGMA0(a) + Maj(a,b,c)
39 // h = g
40 // g = f
41 // f = e
42 // e = d + T1
43 // d = c
44 // c = b
45 // b = a
46 // a = T1 + T2
47 // }
48 //
49 // H0 = a + H0
50 // H1 = b + H1
51 // H2 = c + H2
52 // H3 = d + H3
53 // H4 = e + H4
54 // H5 = f + H5
55 // H6 = g + H6
56 // H7 = h + H7
57
58 // Wt = Mt; for 0 <= t <= 15
59 #define MSGSCHEDULE0(index) \
60 MOVL (index*4)(SI), AX; \
61 BSWAPL AX; \
62 MOVL AX, (index*4)(BP)
63
64 // Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 63
65 // SIGMA0(x) = ROTR(7,x) XOR ROTR(18,x) XOR SHR(3,x)
66 // SIGMA1(x) = ROTR(17,x) XOR ROTR(19,x) XOR SHR(10,x)
67 #define MSGSCHEDULE1(index) \
68 MOVL ((index-2)*4)(BP), AX; \
69 MOVL AX, CX; \
70 RORL $17, AX; \
71 MOVL CX, DX; \
72 RORL $19, CX; \
73 SHRL $10, DX; \
74 MOVL ((index-15)*4)(BP), BX; \
75 XORL CX, AX; \
76 MOVL BX, CX; \
77 XORL DX, AX; \
78 RORL $7, BX; \
79 MOVL CX, DX; \
80 SHRL $3, DX; \
81 RORL $18, CX; \
82 ADDL ((index-7)*4)(BP), AX; \
83 XORL CX, BX; \
84 XORL DX, BX; \
85 ADDL ((index-16)*4)(BP), BX; \
86 ADDL BX, AX; \
87 MOVL AX, ((index)*4)(BP)
88
89 // Calculate T1 in AX - uses AX, CX and DX registers.
90 // h is also used as an accumulator. Wt is passed in AX.
91 // T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt
92 // BIGSIGMA1(x) = ROTR(6,x) XOR ROTR(11,x) XOR ROTR(25,x)
93 // Ch(x, y, z) = (x AND y) XOR (NOT x AND z)
94 #define SHA256T1(const, e, f, g, h) \
95 ADDL AX, h; \
96 MOVL e, AX; \
97 ADDL $const, h; \
98 MOVL e, CX; \
99 RORL $6, AX; \
100 MOVL e, DX; \
101 RORL $11, CX; \
102 XORL CX, AX; \
103 MOVL e, CX; \
104 RORL $25, DX; \
105 ANDL f, CX; \
106 XORL AX, DX; \
107 MOVL e, AX; \
108 NOTL AX; \
109 ADDL DX, h; \
110 ANDL g, AX; \
111 XORL CX, AX; \
112 ADDL h, AX
113
114 // Calculate T2 in BX - uses BX, CX, DX and DI registers.
115 // T2 = BIGSIGMA0(a) + Maj(a, b, c)
116 // BIGSIGMA0(x) = ROTR(2,x) XOR ROTR(13,x) XOR ROTR(22,x)
117 // Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
118 #define SHA256T2(a, b, c) \
119 MOVL a, DI; \
120 MOVL c, BX; \
121 RORL $2, DI; \
122 MOVL a, DX; \
123 ANDL b, BX; \
124 RORL $13, DX; \
125 MOVL a, CX; \
126 ANDL c, CX; \
127 XORL DX, DI; \
128 XORL CX, BX; \
129 MOVL a, DX; \
130 MOVL b, CX; \
131 RORL $22, DX; \
132 ANDL a, CX; \
133 XORL CX, BX; \
134 XORL DX, DI; \
135 ADDL DI, BX
136
137 // Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
138 // The values for e and a are stored in d and h, ready for rotation.
139 #define SHA256ROUND(index, const, a, b, c, d, e, f, g, h) \
140 SHA256T1(const, e, f, g, h); \
141 SHA256T2(a, b, c); \
142 MOVL BX, h; \
143 ADDL AX, d; \
144 ADDL AX, h
145
146 #define SHA256ROUND0(index, const, a, b, c, d, e, f, g, h) \
147 MSGSCHEDULE0(index); \
148 SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
149
150 #define SHA256ROUND1(index, const, a, b, c, d, e, f, g, h) \
151 MSGSCHEDULE1(index); \
152 SHA256ROUND(index, const, a, b, c, d, e, f, g, h)
153
154
155 // Definitions for AVX2 version
156
157 // addm (mem), reg
158 // Add reg to mem using reg-mem add and store
159 #define addm(P1, P2) \
160 ADDL P2, P1; \
161 MOVL P1, P2
162
163 #define XDWORD0 Y4
164 #define XDWORD1 Y5
165 #define XDWORD2 Y6
166 #define XDWORD3 Y7
167
168 #define XWORD0 X4
169 #define XWORD1 X5
170 #define XWORD2 X6
171 #define XWORD3 X7
172
173 #define XTMP0 Y0
174 #define XTMP1 Y1
175 #define XTMP2 Y2
176 #define XTMP3 Y3
177 #define XTMP4 Y8
178 #define XTMP5 Y11
179
180 #define XFER Y9
181
182 #define BYTE_FLIP_MASK Y13 // mask to convert LE -> BE
183 #define X_BYTE_FLIP_MASK X13
184
185 #define NUM_BYTES DX
186 #define INP DI
187
188 #define CTX SI // Beginning of digest in memory (a, b, c, ... , h)
189
190 #define a AX
191 #define b BX
192 #define c CX
193 #define d R8
194 #define e DX
195 #define f R9
196 #define g R10
197 #define h R11
198
199 #define old_h R11
200
201 #define TBL BP
202
203 #define SRND SI // SRND is same register as CTX
204
205 #define T1 R12
206
207 #define y0 R13
208 #define y1 R14
209 #define y2 R15
210 #define y3 DI
211
212 // Offsets
213 #define XFER_SIZE 2*64*4
214 #define INP_END_SIZE 8
215 #define INP_SIZE 8
216
217 #define _XFER 0
218 #define _INP_END _XFER + XFER_SIZE
219 #define _INP _INP_END + INP_END_SIZE
220 #define STACK_SIZE _INP + INP_SIZE
221
222 #define ROUND_AND_SCHED_N_0(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
223 ; \ // ############################# RND N + 0 ############################//
224 MOVL a, y3; \ // y3 = a // MAJA
225 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
226 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
227 ; \
228 ADDL (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h // disp = k + w
229 ORL c, y3; \ // y3 = a|c // MAJA
230 VPALIGNR $4, XDWORD2, XDWORD3, XTMP0; \ // XTMP0 = W[-7]
231 MOVL f, y2; \ // y2 = f // CH
232 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
233 ; \
234 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
235 XORL g, y2; \ // y2 = f^g // CH
236 VPADDD XDWORD0, XTMP0, XTMP0; \ // XTMP0 = W[-7] + W[-16] // y1 = (e >> 6) // S1
237 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
238 ; \
239 ANDL e, y2; \ // y2 = (f^g)&e // CH
240 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
241 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
242 ADDL h, d; \ // d = k + w + h + d // --
243 ; \
244 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
245 VPALIGNR $4, XDWORD0, XDWORD1, XTMP1; \ // XTMP1 = W[-15]
246 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
247 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
248 ; \
249 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
250 VPSRLD $7, XTMP1, XTMP2; \
251 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
252 MOVL a, T1; \ // T1 = a // MAJB
253 ANDL c, T1; \ // T1 = a&c // MAJB
254 ; \
255 ADDL y0, y2; \ // y2 = S1 + CH // --
256 VPSLLD $(32-7), XTMP1, XTMP3; \
257 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
258 ADDL y1, h; \ // h = k + w + h + S0 // --
259 ; \
260 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // --
261 VPOR XTMP2, XTMP3, XTMP3; \ // XTMP3 = W[-15] ror 7
262 ; \
263 VPSRLD $18, XTMP1, XTMP2; \
264 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
265 ADDL y3, h // h = t1 + S0 + MAJ // --
266
267 #define ROUND_AND_SCHED_N_1(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
268 ; \ // ################################### RND N + 1 ############################
269 ; \
270 MOVL a, y3; \ // y3 = a // MAJA
271 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
272 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
273 ADDL (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h // --
274 ORL c, y3; \ // y3 = a|c // MAJA
275 ; \
276 VPSRLD $3, XTMP1, XTMP4; \ // XTMP4 = W[-15] >> 3
277 MOVL f, y2; \ // y2 = f // CH
278 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
279 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
280 XORL g, y2; \ // y2 = f^g // CH
281 ; \
282 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
283 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
284 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
285 ANDL e, y2; \ // y2 = (f^g)&e // CH
286 ADDL h, d; \ // d = k + w + h + d // --
287 ; \
288 VPSLLD $(32-18), XTMP1, XTMP1; \
289 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
290 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
291 ; \
292 VPXOR XTMP1, XTMP3, XTMP3; \
293 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
294 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
295 ; \
296 VPXOR XTMP2, XTMP3, XTMP3; \ // XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
297 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
298 MOVL a, T1; \ // T1 = a // MAJB
299 ANDL c, T1; \ // T1 = a&c // MAJB
300 ADDL y0, y2; \ // y2 = S1 + CH // --
301 ; \
302 VPXOR XTMP4, XTMP3, XTMP1; \ // XTMP1 = s0
303 VPSHUFD $0xFA, XDWORD3, XTMP2; \ // XTMP2 = W[-2] {BBAA}
304 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
305 ADDL y1, h; \ // h = k + w + h + S0 // --
306 ; \
307 VPADDD XTMP1, XTMP0, XTMP0; \ // XTMP0 = W[-16] + W[-7] + s0
308 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // --
309 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
310 ADDL y3, h; \ // h = t1 + S0 + MAJ // --
311 ; \
312 VPSRLD $10, XTMP2, XTMP4 // XTMP4 = W[-2] >> 10 {BBAA}
313
314 #define ROUND_AND_SCHED_N_2(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
315 ; \ // ################################### RND N + 2 ############################
316 ; \
317 MOVL a, y3; \ // y3 = a // MAJA
318 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
319 ADDL (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h // --
320 ; \
321 VPSRLQ $19, XTMP2, XTMP3; \ // XTMP3 = W[-2] ror 19 {xBxA}
322 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
323 ORL c, y3; \ // y3 = a|c // MAJA
324 MOVL f, y2; \ // y2 = f // CH
325 XORL g, y2; \ // y2 = f^g // CH
326 ; \
327 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
328 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
329 VPSRLQ $17, XTMP2, XTMP2; \ // XTMP2 = W[-2] ror 17 {xBxA}
330 ANDL e, y2; \ // y2 = (f^g)&e // CH
331 ; \
332 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
333 VPXOR XTMP3, XTMP2, XTMP2; \
334 ADDL h, d; \ // d = k + w + h + d // --
335 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
336 ; \
337 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
338 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
339 VPXOR XTMP2, XTMP4, XTMP4; \ // XTMP4 = s1 {xBxA}
340 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
341 ; \
342 VPSHUFB shuff_00BA<>(SB), XTMP4, XTMP4;\ // XTMP4 = s1 {00BA}
343 ; \
344 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
345 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
346 VPADDD XTMP4, XTMP0, XTMP0; \ // XTMP0 = {..., ..., W[1], W[0]}
347 ; \
348 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
349 MOVL a, T1; \ // T1 = a // MAJB
350 ANDL c, T1; \ // T1 = a&c // MAJB
351 ADDL y0, y2; \ // y2 = S1 + CH // --
352 VPSHUFD $80, XTMP0, XTMP2; \ // XTMP2 = W[-2] {DDCC}
353 ; \
354 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
355 ADDL y1, h; \ // h = k + w + h + S0 // --
356 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // --
357 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
358 ; \
359 ADDL y3, h // h = t1 + S0 + MAJ // --
360
361 #define ROUND_AND_SCHED_N_3(disp, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3) \
362 ; \ // ################################### RND N + 3 ############################
363 ; \
364 MOVL a, y3; \ // y3 = a // MAJA
365 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
366 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
367 ADDL (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h // --
368 ORL c, y3; \ // y3 = a|c // MAJA
369 ; \
370 VPSRLD $10, XTMP2, XTMP5; \ // XTMP5 = W[-2] >> 10 {DDCC}
371 MOVL f, y2; \ // y2 = f // CH
372 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
373 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
374 XORL g, y2; \ // y2 = f^g // CH
375 ; \
376 VPSRLQ $19, XTMP2, XTMP3; \ // XTMP3 = W[-2] ror 19 {xDxC}
377 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
378 ANDL e, y2; \ // y2 = (f^g)&e // CH
379 ADDL h, d; \ // d = k + w + h + d // --
380 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
381 ; \
382 VPSRLQ $17, XTMP2, XTMP2; \ // XTMP2 = W[-2] ror 17 {xDxC}
383 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
384 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
385 ; \
386 VPXOR XTMP3, XTMP2, XTMP2; \
387 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
388 ADDL y0, y2; \ // y2 = S1 + CH // --
389 ; \
390 VPXOR XTMP2, XTMP5, XTMP5; \ // XTMP5 = s1 {xDxC}
391 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
392 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // --
393 ; \
394 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
395 ; \
396 VPSHUFB shuff_DC00<>(SB), XTMP5, XTMP5;\ // XTMP5 = s1 {DC00}
397 ; \
398 VPADDD XTMP0, XTMP5, XDWORD0; \ // XDWORD0 = {W[3], W[2], W[1], W[0]}
399 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
400 MOVL a, T1; \ // T1 = a // MAJB
401 ANDL c, T1; \ // T1 = a&c // MAJB
402 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
403 ; \
404 ADDL y1, h; \ // h = k + w + h + S0 // --
405 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
406 ADDL y3, h // h = t1 + S0 + MAJ // --
407
408 #define DO_ROUND_N_0(disp, a, b, c, d, e, f, g, h, old_h) \
409 ; \ // ################################### RND N + 0 ###########################
410 MOVL f, y2; \ // y2 = f // CH
411 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
412 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
413 XORL g, y2; \ // y2 = f^g // CH
414 ; \
415 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
416 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
417 ANDL e, y2; \ // y2 = (f^g)&e // CH
418 ; \
419 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
420 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
421 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
422 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
423 MOVL a, y3; \ // y3 = a // MAJA
424 ; \
425 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
426 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
427 ADDL (disp + 0*4)(SP)(SRND*1), h; \ // h = k + w + h // --
428 ORL c, y3; \ // y3 = a|c // MAJA
429 ; \
430 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
431 MOVL a, T1; \ // T1 = a // MAJB
432 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
433 ANDL c, T1; \ // T1 = a&c // MAJB
434 ADDL y0, y2; \ // y2 = S1 + CH // --
435 ; \
436 ADDL h, d; \ // d = k + w + h + d // --
437 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
438 ADDL y1, h; \ // h = k + w + h + S0 // --
439 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // --
440
441 #define DO_ROUND_N_1(disp, a, b, c, d, e, f, g, h, old_h) \
442 ; \ // ################################### RND N + 1 ###########################
443 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0 // --
444 MOVL f, y2; \ // y2 = f // CH
445 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
446 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
447 XORL g, y2; \ // y2 = f^g // CH
448 ; \
449 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
450 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
451 ANDL e, y2; \ // y2 = (f^g)&e // CH
452 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // --
453 ; \
454 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
455 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
456 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
457 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
458 MOVL a, y3; \ // y3 = a // MAJA
459 ; \
460 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
461 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
462 ADDL (disp + 1*4)(SP)(SRND*1), h; \ // h = k + w + h // --
463 ORL c, y3; \ // y3 = a|c // MAJA
464 ; \
465 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
466 MOVL a, T1; \ // T1 = a // MAJB
467 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
468 ANDL c, T1; \ // T1 = a&c // MAJB
469 ADDL y0, y2; \ // y2 = S1 + CH // --
470 ; \
471 ADDL h, d; \ // d = k + w + h + d // --
472 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
473 ADDL y1, h; \ // h = k + w + h + S0 // --
474 ; \
475 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // --
476
477 #define DO_ROUND_N_2(disp, a, b, c, d, e, f, g, h, old_h) \
478 ; \ // ################################### RND N + 2 ##############################
479 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
480 MOVL f, y2; \ // y2 = f // CH
481 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
482 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
483 XORL g, y2; \ // y2 = f^g // CH
484 ; \
485 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
486 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
487 ANDL e, y2; \ // y2 = (f^g)&e // CH
488 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // --
489 ; \
490 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
491 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
492 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
493 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
494 MOVL a, y3; \ // y3 = a // MAJA
495 ; \
496 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
497 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
498 ADDL (disp + 2*4)(SP)(SRND*1), h; \ // h = k + w + h // --
499 ORL c, y3; \ // y3 = a|c // MAJA
500 ; \
501 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
502 MOVL a, T1; \ // T1 = a // MAJB
503 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
504 ANDL c, T1; \ // T1 = a&c // MAJB
505 ADDL y0, y2; \ // y2 = S1 + CH // --
506 ; \
507 ADDL h, d; \ // d = k + w + h + d // --
508 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
509 ADDL y1, h; \ // h = k + w + h + S0 // --
510 ; \
511 ADDL y2, d // d = k + w + h + d + S1 + CH = d + t1 // --
512
513 #define DO_ROUND_N_3(disp, a, b, c, d, e, f, g, h, old_h) \
514 ; \ // ################################### RND N + 3 ###########################
515 ADDL y2, old_h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
516 MOVL f, y2; \ // y2 = f // CH
517 RORXL $25, e, y0; \ // y0 = e >> 25 // S1A
518 RORXL $11, e, y1; \ // y1 = e >> 11 // S1B
519 XORL g, y2; \ // y2 = f^g // CH
520 ; \
521 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) // S1
522 RORXL $6, e, y1; \ // y1 = (e >> 6) // S1
523 ANDL e, y2; \ // y2 = (f^g)&e // CH
524 ADDL y3, old_h; \ // h = t1 + S0 + MAJ // --
525 ; \
526 XORL y1, y0; \ // y0 = (e>>25) ^ (e>>11) ^ (e>>6) // S1
527 RORXL $13, a, T1; \ // T1 = a >> 13 // S0B
528 XORL g, y2; \ // y2 = CH = ((f^g)&e)^g // CH
529 RORXL $22, a, y1; \ // y1 = a >> 22 // S0A
530 MOVL a, y3; \ // y3 = a // MAJA
531 ; \
532 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) // S0
533 RORXL $2, a, T1; \ // T1 = (a >> 2) // S0
534 ADDL (disp + 3*4)(SP)(SRND*1), h; \ // h = k + w + h // --
535 ORL c, y3; \ // y3 = a|c // MAJA
536 ; \
537 XORL T1, y1; \ // y1 = (a>>22) ^ (a>>13) ^ (a>>2) // S0
538 MOVL a, T1; \ // T1 = a // MAJB
539 ANDL b, y3; \ // y3 = (a|c)&b // MAJA
540 ANDL c, T1; \ // T1 = a&c // MAJB
541 ADDL y0, y2; \ // y2 = S1 + CH // --
542 ; \
543 ADDL h, d; \ // d = k + w + h + d // --
544 ORL T1, y3; \ // y3 = MAJ = (a|c)&b)|(a&c) // MAJ
545 ADDL y1, h; \ // h = k + w + h + S0 // --
546 ; \
547 ADDL y2, d; \ // d = k + w + h + d + S1 + CH = d + t1 // --
548 ; \
549 ADDL y2, h; \ // h = k + w + h + S0 + S1 + CH = t1 + S0// --
550 ; \
551 ADDL y3, h // h = t1 + S0 + MAJ // --
552
553 TEXT ·block(SB), 0, $536-32
554 CMPB ·useAVX2(SB), $1
555 JE avx2
556
557 MOVQ p_base+8(FP), SI
558 MOVQ p_len+16(FP), DX
559 SHRQ $6, DX
560 SHLQ $6, DX
561
562 LEAQ (SI)(DX*1), DI
563 MOVQ DI, 256(SP)
564 CMPQ SI, DI
565 JEQ end
566
567 MOVQ dig+0(FP), BP
568 MOVL (0*4)(BP), R8 // a = H0
569 MOVL (1*4)(BP), R9 // b = H1
570 MOVL (2*4)(BP), R10 // c = H2
571 MOVL (3*4)(BP), R11 // d = H3
572 MOVL (4*4)(BP), R12 // e = H4
573 MOVL (5*4)(BP), R13 // f = H5
574 MOVL (6*4)(BP), R14 // g = H6
575 MOVL (7*4)(BP), R15 // h = H7
576
577 loop:
578 MOVQ SP, BP
579
580 SHA256ROUND0(0, 0x428a2f98, R8, R9, R10, R11, R12, R13, R14, R15)
581 SHA256ROUND0(1, 0x71374491, R15, R8, R9, R10, R11, R12, R13, R14)
582 SHA256ROUND0(2, 0xb5c0fbcf, R14, R15, R8, R9, R10, R11, R12, R13)
583 SHA256ROUND0(3, 0xe9b5dba5, R13, R14, R15, R8, R9, R10, R11, R12)
584 SHA256ROUND0(4, 0x3956c25b, R12, R13, R14, R15, R8, R9, R10, R11)
585 SHA256ROUND0(5, 0x59f111f1, R11, R12, R13, R14, R15, R8, R9, R10)
586 SHA256ROUND0(6, 0x923f82a4, R10, R11, R12, R13, R14, R15, R8, R9)
587 SHA256ROUND0(7, 0xab1c5ed5, R9, R10, R11, R12, R13, R14, R15, R8)
588 SHA256ROUND0(8, 0xd807aa98, R8, R9, R10, R11, R12, R13, R14, R15)
589 SHA256ROUND0(9, 0x12835b01, R15, R8, R9, R10, R11, R12, R13, R14)
590 SHA256ROUND0(10, 0x243185be, R14, R15, R8, R9, R10, R11, R12, R13)
591 SHA256ROUND0(11, 0x550c7dc3, R13, R14, R15, R8, R9, R10, R11, R12)
592 SHA256ROUND0(12, 0x72be5d74, R12, R13, R14, R15, R8, R9, R10, R11)
593 SHA256ROUND0(13, 0x80deb1fe, R11, R12, R13, R14, R15, R8, R9, R10)
594 SHA256ROUND0(14, 0x9bdc06a7, R10, R11, R12, R13, R14, R15, R8, R9)
595 SHA256ROUND0(15, 0xc19bf174, R9, R10, R11, R12, R13, R14, R15, R8)
596
597 SHA256ROUND1(16, 0xe49b69c1, R8, R9, R10, R11, R12, R13, R14, R15)
598 SHA256ROUND1(17, 0xefbe4786, R15, R8, R9, R10, R11, R12, R13, R14)
599 SHA256ROUND1(18, 0x0fc19dc6, R14, R15, R8, R9, R10, R11, R12, R13)
600 SHA256ROUND1(19, 0x240ca1cc, R13, R14, R15, R8, R9, R10, R11, R12)
601 SHA256ROUND1(20, 0x2de92c6f, R12, R13, R14, R15, R8, R9, R10, R11)
602 SHA256ROUND1(21, 0x4a7484aa, R11, R12, R13, R14, R15, R8, R9, R10)
603 SHA256ROUND1(22, 0x5cb0a9dc, R10, R11, R12, R13, R14, R15, R8, R9)
604 SHA256ROUND1(23, 0x76f988da, R9, R10, R11, R12, R13, R14, R15, R8)
605 SHA256ROUND1(24, 0x983e5152, R8, R9, R10, R11, R12, R13, R14, R15)
606 SHA256ROUND1(25, 0xa831c66d, R15, R8, R9, R10, R11, R12, R13, R14)
607 SHA256ROUND1(26, 0xb00327c8, R14, R15, R8, R9, R10, R11, R12, R13)
608 SHA256ROUND1(27, 0xbf597fc7, R13, R14, R15, R8, R9, R10, R11, R12)
609 SHA256ROUND1(28, 0xc6e00bf3, R12, R13, R14, R15, R8, R9, R10, R11)
610 SHA256ROUND1(29, 0xd5a79147, R11, R12, R13, R14, R15, R8, R9, R10)
611 SHA256ROUND1(30, 0x06ca6351, R10, R11, R12, R13, R14, R15, R8, R9)
612 SHA256ROUND1(31, 0x14292967, R9, R10, R11, R12, R13, R14, R15, R8)
613 SHA256ROUND1(32, 0x27b70a85, R8, R9, R10, R11, R12, R13, R14, R15)
614 SHA256ROUND1(33, 0x2e1b2138, R15, R8, R9, R10, R11, R12, R13, R14)
615 SHA256ROUND1(34, 0x4d2c6dfc, R14, R15, R8, R9, R10, R11, R12, R13)
616 SHA256ROUND1(35, 0x53380d13, R13, R14, R15, R8, R9, R10, R11, R12)
617 SHA256ROUND1(36, 0x650a7354, R12, R13, R14, R15, R8, R9, R10, R11)
618 SHA256ROUND1(37, 0x766a0abb, R11, R12, R13, R14, R15, R8, R9, R10)
619 SHA256ROUND1(38, 0x81c2c92e, R10, R11, R12, R13, R14, R15, R8, R9)
620 SHA256ROUND1(39, 0x92722c85, R9, R10, R11, R12, R13, R14, R15, R8)
621 SHA256ROUND1(40, 0xa2bfe8a1, R8, R9, R10, R11, R12, R13, R14, R15)
622 SHA256ROUND1(41, 0xa81a664b, R15, R8, R9, R10, R11, R12, R13, R14)
623 SHA256ROUND1(42, 0xc24b8b70, R14, R15, R8, R9, R10, R11, R12, R13)
624 SHA256ROUND1(43, 0xc76c51a3, R13, R14, R15, R8, R9, R10, R11, R12)
625 SHA256ROUND1(44, 0xd192e819, R12, R13, R14, R15, R8, R9, R10, R11)
626 SHA256ROUND1(45, 0xd6990624, R11, R12, R13, R14, R15, R8, R9, R10)
627 SHA256ROUND1(46, 0xf40e3585, R10, R11, R12, R13, R14, R15, R8, R9)
628 SHA256ROUND1(47, 0x106aa070, R9, R10, R11, R12, R13, R14, R15, R8)
629 SHA256ROUND1(48, 0x19a4c116, R8, R9, R10, R11, R12, R13, R14, R15)
630 SHA256ROUND1(49, 0x1e376c08, R15, R8, R9, R10, R11, R12, R13, R14)
631 SHA256ROUND1(50, 0x2748774c, R14, R15, R8, R9, R10, R11, R12, R13)
632 SHA256ROUND1(51, 0x34b0bcb5, R13, R14, R15, R8, R9, R10, R11, R12)
633 SHA256ROUND1(52, 0x391c0cb3, R12, R13, R14, R15, R8, R9, R10, R11)
634 SHA256ROUND1(53, 0x4ed8aa4a, R11, R12, R13, R14, R15, R8, R9, R10)
635 SHA256ROUND1(54, 0x5b9cca4f, R10, R11, R12, R13, R14, R15, R8, R9)
636 SHA256ROUND1(55, 0x682e6ff3, R9, R10, R11, R12, R13, R14, R15, R8)
637 SHA256ROUND1(56, 0x748f82ee, R8, R9, R10, R11, R12, R13, R14, R15)
638 SHA256ROUND1(57, 0x78a5636f, R15, R8, R9, R10, R11, R12, R13, R14)
639 SHA256ROUND1(58, 0x84c87814, R14, R15, R8, R9, R10, R11, R12, R13)
640 SHA256ROUND1(59, 0x8cc70208, R13, R14, R15, R8, R9, R10, R11, R12)
641 SHA256ROUND1(60, 0x90befffa, R12, R13, R14, R15, R8, R9, R10, R11)
642 SHA256ROUND1(61, 0xa4506ceb, R11, R12, R13, R14, R15, R8, R9, R10)
643 SHA256ROUND1(62, 0xbef9a3f7, R10, R11, R12, R13, R14, R15, R8, R9)
644 SHA256ROUND1(63, 0xc67178f2, R9, R10, R11, R12, R13, R14, R15, R8)
645
646 MOVQ dig+0(FP), BP
647 ADDL (0*4)(BP), R8 // H0 = a + H0
648 MOVL R8, (0*4)(BP)
649 ADDL (1*4)(BP), R9 // H1 = b + H1
650 MOVL R9, (1*4)(BP)
651 ADDL (2*4)(BP), R10 // H2 = c + H2
652 MOVL R10, (2*4)(BP)
653 ADDL (3*4)(BP), R11 // H3 = d + H3
654 MOVL R11, (3*4)(BP)
655 ADDL (4*4)(BP), R12 // H4 = e + H4
656 MOVL R12, (4*4)(BP)
657 ADDL (5*4)(BP), R13 // H5 = f + H5
658 MOVL R13, (5*4)(BP)
659 ADDL (6*4)(BP), R14 // H6 = g + H6
660 MOVL R14, (6*4)(BP)
661 ADDL (7*4)(BP), R15 // H7 = h + H7
662 MOVL R15, (7*4)(BP)
663
664 ADDQ $64, SI
665 CMPQ SI, 256(SP)
666 JB loop
667
668 end:
669 RET
670
671 avx2:
672 MOVQ dig+0(FP), CTX // d.h[8]
673 MOVQ p_base+8(FP), INP
674 MOVQ p_len+16(FP), NUM_BYTES
675
676 LEAQ -64(INP)(NUM_BYTES*1), NUM_BYTES // Pointer to the last block
677 MOVQ NUM_BYTES, _INP_END(SP)
678
679 CMPQ NUM_BYTES, INP
680 JE avx2_only_one_block
681
682 // Load initial digest
683 MOVL 0(CTX), a // a = H0
684 MOVL 4(CTX), b // b = H1
685 MOVL 8(CTX), c // c = H2
686 MOVL 12(CTX), d // d = H3
687 MOVL 16(CTX), e // e = H4
688 MOVL 20(CTX), f // f = H5
689 MOVL 24(CTX), g // g = H6
690 MOVL 28(CTX), h // h = H7
691
692 avx2_loop0: // at each iteration works with one block (512 bit)
693
694 VMOVDQU (0*32)(INP), XTMP0
695 VMOVDQU (1*32)(INP), XTMP1
696 VMOVDQU (2*32)(INP), XTMP2
697 VMOVDQU (3*32)(INP), XTMP3
698
699 VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
700
701 // Apply Byte Flip Mask: LE -> BE
702 VPSHUFB BYTE_FLIP_MASK, XTMP0, XTMP0
703 VPSHUFB BYTE_FLIP_MASK, XTMP1, XTMP1
704 VPSHUFB BYTE_FLIP_MASK, XTMP2, XTMP2
705 VPSHUFB BYTE_FLIP_MASK, XTMP3, XTMP3
706
707 // Transpose data into high/low parts
708 VPERM2I128 $0x20, XTMP2, XTMP0, XDWORD0 // w3, w2, w1, w0
709 VPERM2I128 $0x31, XTMP2, XTMP0, XDWORD1 // w7, w6, w5, w4
710 VPERM2I128 $0x20, XTMP3, XTMP1, XDWORD2 // w11, w10, w9, w8
711 VPERM2I128 $0x31, XTMP3, XTMP1, XDWORD3 // w15, w14, w13, w12
712
713 MOVQ $K256<>(SB), TBL // Loading address of table with round-specific constants
714
715 avx2_last_block_enter:
716 ADDQ $64, INP
717 MOVQ INP, _INP(SP)
718 XORQ SRND, SRND
719
720 avx2_loop1: // for w0 - w47
721 // Do 4 rounds and scheduling
722 VPADDD 0*32(TBL)(SRND*1), XDWORD0, XFER
723 VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1)
724 ROUND_AND_SCHED_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
725 ROUND_AND_SCHED_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
726 ROUND_AND_SCHED_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
727 ROUND_AND_SCHED_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, XDWORD0, XDWORD1, XDWORD2, XDWORD3)
728
729 // Do 4 rounds and scheduling
730 VPADDD 1*32(TBL)(SRND*1), XDWORD1, XFER
731 VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1)
732 ROUND_AND_SCHED_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
733 ROUND_AND_SCHED_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
734 ROUND_AND_SCHED_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
735 ROUND_AND_SCHED_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, XDWORD1, XDWORD2, XDWORD3, XDWORD0)
736
737 // Do 4 rounds and scheduling
738 VPADDD 2*32(TBL)(SRND*1), XDWORD2, XFER
739 VMOVDQU XFER, (_XFER + 2*32)(SP)(SRND*1)
740 ROUND_AND_SCHED_N_0(_XFER + 2*32, a, b, c, d, e, f, g, h, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
741 ROUND_AND_SCHED_N_1(_XFER + 2*32, h, a, b, c, d, e, f, g, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
742 ROUND_AND_SCHED_N_2(_XFER + 2*32, g, h, a, b, c, d, e, f, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
743 ROUND_AND_SCHED_N_3(_XFER + 2*32, f, g, h, a, b, c, d, e, XDWORD2, XDWORD3, XDWORD0, XDWORD1)
744
745 // Do 4 rounds and scheduling
746 VPADDD 3*32(TBL)(SRND*1), XDWORD3, XFER
747 VMOVDQU XFER, (_XFER + 3*32)(SP)(SRND*1)
748 ROUND_AND_SCHED_N_0(_XFER + 3*32, e, f, g, h, a, b, c, d, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
749 ROUND_AND_SCHED_N_1(_XFER + 3*32, d, e, f, g, h, a, b, c, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
750 ROUND_AND_SCHED_N_2(_XFER + 3*32, c, d, e, f, g, h, a, b, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
751 ROUND_AND_SCHED_N_3(_XFER + 3*32, b, c, d, e, f, g, h, a, XDWORD3, XDWORD0, XDWORD1, XDWORD2)
752
753 ADDQ $4*32, SRND
754 CMPQ SRND, $3*4*32
755 JB avx2_loop1
756
757 avx2_loop2:
758 // w48 - w63 processed with no scheduling (last 16 rounds)
759 VPADDD 0*32(TBL)(SRND*1), XDWORD0, XFER
760 VMOVDQU XFER, (_XFER + 0*32)(SP)(SRND*1)
761 DO_ROUND_N_0(_XFER + 0*32, a, b, c, d, e, f, g, h, h)
762 DO_ROUND_N_1(_XFER + 0*32, h, a, b, c, d, e, f, g, h)
763 DO_ROUND_N_2(_XFER + 0*32, g, h, a, b, c, d, e, f, g)
764 DO_ROUND_N_3(_XFER + 0*32, f, g, h, a, b, c, d, e, f)
765
766 VPADDD 1*32(TBL)(SRND*1), XDWORD1, XFER
767 VMOVDQU XFER, (_XFER + 1*32)(SP)(SRND*1)
768 DO_ROUND_N_0(_XFER + 1*32, e, f, g, h, a, b, c, d, e)
769 DO_ROUND_N_1(_XFER + 1*32, d, e, f, g, h, a, b, c, d)
770 DO_ROUND_N_2(_XFER + 1*32, c, d, e, f, g, h, a, b, c)
771 DO_ROUND_N_3(_XFER + 1*32, b, c, d, e, f, g, h, a, b)
772
773 ADDQ $2*32, SRND
774
775 VMOVDQU XDWORD2, XDWORD0
776 VMOVDQU XDWORD3, XDWORD1
777
778 CMPQ SRND, $4*4*32
779 JB avx2_loop2
780
781 MOVQ dig+0(FP), CTX // d.h[8]
782 MOVQ _INP(SP), INP
783
784 addm( 0(CTX), a)
785 addm( 4(CTX), b)
786 addm( 8(CTX), c)
787 addm( 12(CTX), d)
788 addm( 16(CTX), e)
789 addm( 20(CTX), f)
790 addm( 24(CTX), g)
791 addm( 28(CTX), h)
792
793 CMPQ _INP_END(SP), INP
794 JB done_hash
795
796 XORQ SRND, SRND
797
798 avx2_loop3: // Do second block using previously scheduled results
799 DO_ROUND_N_0(_XFER + 0*32 + 16, a, b, c, d, e, f, g, h, a)
800 DO_ROUND_N_1(_XFER + 0*32 + 16, h, a, b, c, d, e, f, g, h)
801 DO_ROUND_N_2(_XFER + 0*32 + 16, g, h, a, b, c, d, e, f, g)
802 DO_ROUND_N_3(_XFER + 0*32 + 16, f, g, h, a, b, c, d, e, f)
803
804 DO_ROUND_N_0(_XFER + 1*32 + 16, e, f, g, h, a, b, c, d, e)
805 DO_ROUND_N_1(_XFER + 1*32 + 16, d, e, f, g, h, a, b, c, d)
806 DO_ROUND_N_2(_XFER + 1*32 + 16, c, d, e, f, g, h, a, b, c)
807 DO_ROUND_N_3(_XFER + 1*32 + 16, b, c, d, e, f, g, h, a, b)
808
809 ADDQ $2*32, SRND
810 CMPQ SRND, $4*4*32
811 JB avx2_loop3
812
813 MOVQ dig+0(FP), CTX // d.h[8]
814 MOVQ _INP(SP), INP
815 ADDQ $64, INP
816
817 addm( 0(CTX), a)
818 addm( 4(CTX), b)
819 addm( 8(CTX), c)
820 addm( 12(CTX), d)
821 addm( 16(CTX), e)
822 addm( 20(CTX), f)
823 addm( 24(CTX), g)
824 addm( 28(CTX), h)
825
826 CMPQ _INP_END(SP), INP
827 JA avx2_loop0
828 JB done_hash
829
830 avx2_do_last_block:
831
832 VMOVDQU 0(INP), XWORD0
833 VMOVDQU 16(INP), XWORD1
834 VMOVDQU 32(INP), XWORD2
835 VMOVDQU 48(INP), XWORD3
836
837 VMOVDQU flip_mask<>(SB), BYTE_FLIP_MASK
838
839 VPSHUFB X_BYTE_FLIP_MASK, XWORD0, XWORD0
840 VPSHUFB X_BYTE_FLIP_MASK, XWORD1, XWORD1
841 VPSHUFB X_BYTE_FLIP_MASK, XWORD2, XWORD2
842 VPSHUFB X_BYTE_FLIP_MASK, XWORD3, XWORD3
843
844 MOVQ $K256<>(SB), TBL
845
846 JMP avx2_last_block_enter
847
848 avx2_only_one_block:
849 // Load initial digest
850 MOVL 0(CTX), a // a = H0
851 MOVL 4(CTX), b // b = H1
852 MOVL 8(CTX), c // c = H2
853 MOVL 12(CTX), d // d = H3
854 MOVL 16(CTX), e // e = H4
855 MOVL 20(CTX), f // f = H5
856 MOVL 24(CTX), g // g = H6
857 MOVL 28(CTX), h // h = H7
858
859 JMP avx2_do_last_block
860
861 done_hash:
862 VZEROUPPER
863 RET
864
865 // shuffle byte order from LE to BE
866 DATA flip_mask<>+0x00(SB)/8, $0x0405060700010203
867 DATA flip_mask<>+0x08(SB)/8, $0x0c0d0e0f08090a0b
868 DATA flip_mask<>+0x10(SB)/8, $0x0405060700010203
869 DATA flip_mask<>+0x18(SB)/8, $0x0c0d0e0f08090a0b
870 GLOBL flip_mask<>(SB), 8, $32
871
872 // shuffle xBxA -> 00BA
873 DATA shuff_00BA<>+0x00(SB)/8, $0x0b0a090803020100
874 DATA shuff_00BA<>+0x08(SB)/8, $0xFFFFFFFFFFFFFFFF
875 DATA shuff_00BA<>+0x10(SB)/8, $0x0b0a090803020100
876 DATA shuff_00BA<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
877 GLOBL shuff_00BA<>(SB), 8, $32
878
879 // shuffle xDxC -> DC00
880 DATA shuff_DC00<>+0x00(SB)/8, $0xFFFFFFFFFFFFFFFF
881 DATA shuff_DC00<>+0x08(SB)/8, $0x0b0a090803020100
882 DATA shuff_DC00<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
883 DATA shuff_DC00<>+0x18(SB)/8, $0x0b0a090803020100
884 GLOBL shuff_DC00<>(SB), 8, $32
885
886 // Round specific constants
887 DATA K256<>+0x00(SB)/4, $0x428a2f98 // k1
888 DATA K256<>+0x04(SB)/4, $0x71374491 // k2
889 DATA K256<>+0x08(SB)/4, $0xb5c0fbcf // k3
890 DATA K256<>+0x0c(SB)/4, $0xe9b5dba5 // k4
891 DATA K256<>+0x10(SB)/4, $0x428a2f98 // k1
892 DATA K256<>+0x14(SB)/4, $0x71374491 // k2
893 DATA K256<>+0x18(SB)/4, $0xb5c0fbcf // k3
894 DATA K256<>+0x1c(SB)/4, $0xe9b5dba5 // k4
895
896 DATA K256<>+0x20(SB)/4, $0x3956c25b // k5 - k8
897 DATA K256<>+0x24(SB)/4, $0x59f111f1
898 DATA K256<>+0x28(SB)/4, $0x923f82a4
899 DATA K256<>+0x2c(SB)/4, $0xab1c5ed5
900 DATA K256<>+0x30(SB)/4, $0x3956c25b
901 DATA K256<>+0x34(SB)/4, $0x59f111f1
902 DATA K256<>+0x38(SB)/4, $0x923f82a4
903 DATA K256<>+0x3c(SB)/4, $0xab1c5ed5
904
905 DATA K256<>+0x40(SB)/4, $0xd807aa98 // k9 - k12
906 DATA K256<>+0x44(SB)/4, $0x12835b01
907 DATA K256<>+0x48(SB)/4, $0x243185be
908 DATA K256<>+0x4c(SB)/4, $0x550c7dc3
909 DATA K256<>+0x50(SB)/4, $0xd807aa98
910 DATA K256<>+0x54(SB)/4, $0x12835b01
911 DATA K256<>+0x58(SB)/4, $0x243185be
912 DATA K256<>+0x5c(SB)/4, $0x550c7dc3
913
914 DATA K256<>+0x60(SB)/4, $0x72be5d74 // k13 - k16
915 DATA K256<>+0x64(SB)/4, $0x80deb1fe
916 DATA K256<>+0x68(SB)/4, $0x9bdc06a7
917 DATA K256<>+0x6c(SB)/4, $0xc19bf174
918 DATA K256<>+0x70(SB)/4, $0x72be5d74
919 DATA K256<>+0x74(SB)/4, $0x80deb1fe
920 DATA K256<>+0x78(SB)/4, $0x9bdc06a7
921 DATA K256<>+0x7c(SB)/4, $0xc19bf174
922
923 DATA K256<>+0x80(SB)/4, $0xe49b69c1 // k17 - k20
924 DATA K256<>+0x84(SB)/4, $0xefbe4786
925 DATA K256<>+0x88(SB)/4, $0x0fc19dc6
926 DATA K256<>+0x8c(SB)/4, $0x240ca1cc
927 DATA K256<>+0x90(SB)/4, $0xe49b69c1
928 DATA K256<>+0x94(SB)/4, $0xefbe4786
929 DATA K256<>+0x98(SB)/4, $0x0fc19dc6
930 DATA K256<>+0x9c(SB)/4, $0x240ca1cc
931
932 DATA K256<>+0xa0(SB)/4, $0x2de92c6f // k21 - k24
933 DATA K256<>+0xa4(SB)/4, $0x4a7484aa
934 DATA K256<>+0xa8(SB)/4, $0x5cb0a9dc
935 DATA K256<>+0xac(SB)/4, $0x76f988da
936 DATA K256<>+0xb0(SB)/4, $0x2de92c6f
937 DATA K256<>+0xb4(SB)/4, $0x4a7484aa
938 DATA K256<>+0xb8(SB)/4, $0x5cb0a9dc
939 DATA K256<>+0xbc(SB)/4, $0x76f988da
940
941 DATA K256<>+0xc0(SB)/4, $0x983e5152 // k25 - k28
942 DATA K256<>+0xc4(SB)/4, $0xa831c66d
943 DATA K256<>+0xc8(SB)/4, $0xb00327c8
944 DATA K256<>+0xcc(SB)/4, $0xbf597fc7
945 DATA K256<>+0xd0(SB)/4, $0x983e5152
946 DATA K256<>+0xd4(SB)/4, $0xa831c66d
947 DATA K256<>+0xd8(SB)/4, $0xb00327c8
948 DATA K256<>+0xdc(SB)/4, $0xbf597fc7
949
950 DATA K256<>+0xe0(SB)/4, $0xc6e00bf3 // k29 - k32
951 DATA K256<>+0xe4(SB)/4, $0xd5a79147
952 DATA K256<>+0xe8(SB)/4, $0x06ca6351
953 DATA K256<>+0xec(SB)/4, $0x14292967
954 DATA K256<>+0xf0(SB)/4, $0xc6e00bf3
955 DATA K256<>+0xf4(SB)/4, $0xd5a79147
956 DATA K256<>+0xf8(SB)/4, $0x06ca6351
957 DATA K256<>+0xfc(SB)/4, $0x14292967
958
959 DATA K256<>+0x100(SB)/4, $0x27b70a85
960 DATA K256<>+0x104(SB)/4, $0x2e1b2138
961 DATA K256<>+0x108(SB)/4, $0x4d2c6dfc
962 DATA K256<>+0x10c(SB)/4, $0x53380d13
963 DATA K256<>+0x110(SB)/4, $0x27b70a85
964 DATA K256<>+0x114(SB)/4, $0x2e1b2138
965 DATA K256<>+0x118(SB)/4, $0x4d2c6dfc
966 DATA K256<>+0x11c(SB)/4, $0x53380d13
967
968 DATA K256<>+0x120(SB)/4, $0x650a7354
969 DATA K256<>+0x124(SB)/4, $0x766a0abb
970 DATA K256<>+0x128(SB)/4, $0x81c2c92e
971 DATA K256<>+0x12c(SB)/4, $0x92722c85
972 DATA K256<>+0x130(SB)/4, $0x650a7354
973 DATA K256<>+0x134(SB)/4, $0x766a0abb
974 DATA K256<>+0x138(SB)/4, $0x81c2c92e
975 DATA K256<>+0x13c(SB)/4, $0x92722c85
976
977 DATA K256<>+0x140(SB)/4, $0xa2bfe8a1
978 DATA K256<>+0x144(SB)/4, $0xa81a664b
979 DATA K256<>+0x148(SB)/4, $0xc24b8b70
980 DATA K256<>+0x14c(SB)/4, $0xc76c51a3
981 DATA K256<>+0x150(SB)/4, $0xa2bfe8a1
982 DATA K256<>+0x154(SB)/4, $0xa81a664b
983 DATA K256<>+0x158(SB)/4, $0xc24b8b70
984 DATA K256<>+0x15c(SB)/4, $0xc76c51a3
985
986 DATA K256<>+0x160(SB)/4, $0xd192e819
987 DATA K256<>+0x164(SB)/4, $0xd6990624
988 DATA K256<>+0x168(SB)/4, $0xf40e3585
989 DATA K256<>+0x16c(SB)/4, $0x106aa070
990 DATA K256<>+0x170(SB)/4, $0xd192e819
991 DATA K256<>+0x174(SB)/4, $0xd6990624
992 DATA K256<>+0x178(SB)/4, $0xf40e3585
993 DATA K256<>+0x17c(SB)/4, $0x106aa070
994
995 DATA K256<>+0x180(SB)/4, $0x19a4c116
996 DATA K256<>+0x184(SB)/4, $0x1e376c08
997 DATA K256<>+0x188(SB)/4, $0x2748774c
998 DATA K256<>+0x18c(SB)/4, $0x34b0bcb5
999 DATA K256<>+0x190(SB)/4, $0x19a4c116
1000 DATA K256<>+0x194(SB)/4, $0x1e376c08
1001 DATA K256<>+0x198(SB)/4, $0x2748774c
1002 DATA K256<>+0x19c(SB)/4, $0x34b0bcb5
1003
1004 DATA K256<>+0x1a0(SB)/4, $0x391c0cb3
1005 DATA K256<>+0x1a4(SB)/4, $0x4ed8aa4a
1006 DATA K256<>+0x1a8(SB)/4, $0x5b9cca4f
1007 DATA K256<>+0x1ac(SB)/4, $0x682e6ff3
1008 DATA K256<>+0x1b0(SB)/4, $0x391c0cb3
1009 DATA K256<>+0x1b4(SB)/4, $0x4ed8aa4a
1010 DATA K256<>+0x1b8(SB)/4, $0x5b9cca4f
1011 DATA K256<>+0x1bc(SB)/4, $0x682e6ff3
1012
1013 DATA K256<>+0x1c0(SB)/4, $0x748f82ee
1014 DATA K256<>+0x1c4(SB)/4, $0x78a5636f
1015 DATA K256<>+0x1c8(SB)/4, $0x84c87814
1016 DATA K256<>+0x1cc(SB)/4, $0x8cc70208
1017 DATA K256<>+0x1d0(SB)/4, $0x748f82ee
1018 DATA K256<>+0x1d4(SB)/4, $0x78a5636f
1019 DATA K256<>+0x1d8(SB)/4, $0x84c87814
1020 DATA K256<>+0x1dc(SB)/4, $0x8cc70208
1021
1022 DATA K256<>+0x1e0(SB)/4, $0x90befffa
1023 DATA K256<>+0x1e4(SB)/4, $0xa4506ceb
1024 DATA K256<>+0x1e8(SB)/4, $0xbef9a3f7
1025 DATA K256<>+0x1ec(SB)/4, $0xc67178f2
1026 DATA K256<>+0x1f0(SB)/4, $0x90befffa
1027 DATA K256<>+0x1f4(SB)/4, $0xa4506ceb
1028 DATA K256<>+0x1f8(SB)/4, $0xbef9a3f7
1029 DATA K256<>+0x1fc(SB)/4, $0xc67178f2
1030
1031 GLOBL K256<>(SB), (NOPTR + RODATA), $512
View as plain text