Text file src/crypto/sha512/sha512block_amd64.s
1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "textflag.h"
6
7 // SHA512 block routine. See sha512block.go for Go equivalent.
8 //
9 // The algorithm is detailed in FIPS 180-4:
10 //
11 // https://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf
12 //
13 // Wt = Mt; for 0 <= t <= 15
14 // Wt = SIGMA1(Wt-2) + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79
15 //
16 // a = H0
17 // b = H1
18 // c = H2
19 // d = H3
20 // e = H4
21 // f = H5
22 // g = H6
23 // h = H7
24 //
25 // for t = 0 to 79 {
26 // T1 = h + BIGSIGMA1(e) + Ch(e,f,g) + Kt + Wt
27 // T2 = BIGSIGMA0(a) + Maj(a,b,c)
28 // h = g
29 // g = f
30 // f = e
31 // e = d + T1
32 // d = c
33 // c = b
34 // b = a
35 // a = T1 + T2
36 // }
37 //
38 // H0 = a + H0
39 // H1 = b + H1
40 // H2 = c + H2
41 // H3 = d + H3
42 // H4 = e + H4
43 // H5 = f + H5
44 // H6 = g + H6
45 // H7 = h + H7
46
47 // Wt = Mt; for 0 <= t <= 15
48 #define MSGSCHEDULE0(index) \
49 MOVQ (index*8)(SI), AX; \
50 BSWAPQ AX; \
51 MOVQ AX, (index*8)(BP)
52
53 // Wt = SIGMA1(Wt-2) + Wt-7 + SIGMA0(Wt-15) + Wt-16; for 16 <= t <= 79
54 // SIGMA0(x) = ROTR(1,x) XOR ROTR(8,x) XOR SHR(7,x)
55 // SIGMA1(x) = ROTR(19,x) XOR ROTR(61,x) XOR SHR(6,x)
56 #define MSGSCHEDULE1(index) \
57 MOVQ ((index-2)*8)(BP), AX; \
58 MOVQ AX, CX; \
59 RORQ $19, AX; \
60 MOVQ CX, DX; \
61 RORQ $61, CX; \
62 SHRQ $6, DX; \
63 MOVQ ((index-15)*8)(BP), BX; \
64 XORQ CX, AX; \
65 MOVQ BX, CX; \
66 XORQ DX, AX; \
67 RORQ $1, BX; \
68 MOVQ CX, DX; \
69 SHRQ $7, DX; \
70 RORQ $8, CX; \
71 ADDQ ((index-7)*8)(BP), AX; \
72 XORQ CX, BX; \
73 XORQ DX, BX; \
74 ADDQ ((index-16)*8)(BP), BX; \
75 ADDQ BX, AX; \
76 MOVQ AX, ((index)*8)(BP)
77
78 // Calculate T1 in AX - uses AX, CX and DX registers.
79 // h is also used as an accumulator. Wt is passed in AX.
80 // T1 = h + BIGSIGMA1(e) + Ch(e, f, g) + Kt + Wt
81 // BIGSIGMA1(x) = ROTR(14,x) XOR ROTR(18,x) XOR ROTR(41,x)
82 // Ch(x, y, z) = (x AND y) XOR (NOT x AND z)
83 #define SHA512T1(const, e, f, g, h) \
84 MOVQ $const, DX; \
85 ADDQ AX, h; \
86 MOVQ e, AX; \
87 ADDQ DX, h; \
88 MOVQ e, CX; \
89 RORQ $14, AX; \
90 MOVQ e, DX; \
91 RORQ $18, CX; \
92 XORQ CX, AX; \
93 MOVQ e, CX; \
94 RORQ $41, DX; \
95 ANDQ f, CX; \
96 XORQ AX, DX; \
97 MOVQ e, AX; \
98 NOTQ AX; \
99 ADDQ DX, h; \
100 ANDQ g, AX; \
101 XORQ CX, AX; \
102 ADDQ h, AX
103
104 // Calculate T2 in BX - uses BX, CX, DX and DI registers.
105 // T2 = BIGSIGMA0(a) + Maj(a, b, c)
106 // BIGSIGMA0(x) = ROTR(28,x) XOR ROTR(34,x) XOR ROTR(39,x)
107 // Maj(x, y, z) = (x AND y) XOR (x AND z) XOR (y AND z)
108 #define SHA512T2(a, b, c) \
109 MOVQ a, DI; \
110 MOVQ c, BX; \
111 RORQ $28, DI; \
112 MOVQ a, DX; \
113 ANDQ b, BX; \
114 RORQ $34, DX; \
115 MOVQ a, CX; \
116 ANDQ c, CX; \
117 XORQ DX, DI; \
118 XORQ CX, BX; \
119 MOVQ a, DX; \
120 MOVQ b, CX; \
121 RORQ $39, DX; \
122 ANDQ a, CX; \
123 XORQ CX, BX; \
124 XORQ DX, DI; \
125 ADDQ DI, BX
126
127 // Calculate T1 and T2, then e = d + T1 and a = T1 + T2.
128 // The values for e and a are stored in d and h, ready for rotation.
129 #define SHA512ROUND(index, const, a, b, c, d, e, f, g, h) \
130 SHA512T1(const, e, f, g, h); \
131 SHA512T2(a, b, c); \
132 MOVQ BX, h; \
133 ADDQ AX, d; \
134 ADDQ AX, h
135
136 #define SHA512ROUND0(index, const, a, b, c, d, e, f, g, h) \
137 MSGSCHEDULE0(index); \
138 SHA512ROUND(index, const, a, b, c, d, e, f, g, h)
139
140 #define SHA512ROUND1(index, const, a, b, c, d, e, f, g, h) \
141 MSGSCHEDULE1(index); \
142 SHA512ROUND(index, const, a, b, c, d, e, f, g, h)
143
144 TEXT ·blockAMD64(SB),0,$648-32
145 MOVQ p_base+8(FP), SI
146 MOVQ p_len+16(FP), DX
147 SHRQ $7, DX
148 SHLQ $7, DX
149
150 LEAQ (SI)(DX*1), DI
151 MOVQ DI, 640(SP)
152 CMPQ SI, DI
153 JEQ end
154
155 MOVQ dig+0(FP), BP
156 MOVQ (0*8)(BP), R8 // a = H0
157 MOVQ (1*8)(BP), R9 // b = H1
158 MOVQ (2*8)(BP), R10 // c = H2
159 MOVQ (3*8)(BP), R11 // d = H3
160 MOVQ (4*8)(BP), R12 // e = H4
161 MOVQ (5*8)(BP), R13 // f = H5
162 MOVQ (6*8)(BP), R14 // g = H6
163 MOVQ (7*8)(BP), R15 // h = H7
164
165 loop:
166 MOVQ SP, BP // message schedule
167
168 SHA512ROUND0(0, 0x428a2f98d728ae22, R8, R9, R10, R11, R12, R13, R14, R15)
169 SHA512ROUND0(1, 0x7137449123ef65cd, R15, R8, R9, R10, R11, R12, R13, R14)
170 SHA512ROUND0(2, 0xb5c0fbcfec4d3b2f, R14, R15, R8, R9, R10, R11, R12, R13)
171 SHA512ROUND0(3, 0xe9b5dba58189dbbc, R13, R14, R15, R8, R9, R10, R11, R12)
172 SHA512ROUND0(4, 0x3956c25bf348b538, R12, R13, R14, R15, R8, R9, R10, R11)
173 SHA512ROUND0(5, 0x59f111f1b605d019, R11, R12, R13, R14, R15, R8, R9, R10)
174 SHA512ROUND0(6, 0x923f82a4af194f9b, R10, R11, R12, R13, R14, R15, R8, R9)
175 SHA512ROUND0(7, 0xab1c5ed5da6d8118, R9, R10, R11, R12, R13, R14, R15, R8)
176 SHA512ROUND0(8, 0xd807aa98a3030242, R8, R9, R10, R11, R12, R13, R14, R15)
177 SHA512ROUND0(9, 0x12835b0145706fbe, R15, R8, R9, R10, R11, R12, R13, R14)
178 SHA512ROUND0(10, 0x243185be4ee4b28c, R14, R15, R8, R9, R10, R11, R12, R13)
179 SHA512ROUND0(11, 0x550c7dc3d5ffb4e2, R13, R14, R15, R8, R9, R10, R11, R12)
180 SHA512ROUND0(12, 0x72be5d74f27b896f, R12, R13, R14, R15, R8, R9, R10, R11)
181 SHA512ROUND0(13, 0x80deb1fe3b1696b1, R11, R12, R13, R14, R15, R8, R9, R10)
182 SHA512ROUND0(14, 0x9bdc06a725c71235, R10, R11, R12, R13, R14, R15, R8, R9)
183 SHA512ROUND0(15, 0xc19bf174cf692694, R9, R10, R11, R12, R13, R14, R15, R8)
184
185 SHA512ROUND1(16, 0xe49b69c19ef14ad2, R8, R9, R10, R11, R12, R13, R14, R15)
186 SHA512ROUND1(17, 0xefbe4786384f25e3, R15, R8, R9, R10, R11, R12, R13, R14)
187 SHA512ROUND1(18, 0x0fc19dc68b8cd5b5, R14, R15, R8, R9, R10, R11, R12, R13)
188 SHA512ROUND1(19, 0x240ca1cc77ac9c65, R13, R14, R15, R8, R9, R10, R11, R12)
189 SHA512ROUND1(20, 0x2de92c6f592b0275, R12, R13, R14, R15, R8, R9, R10, R11)
190 SHA512ROUND1(21, 0x4a7484aa6ea6e483, R11, R12, R13, R14, R15, R8, R9, R10)
191 SHA512ROUND1(22, 0x5cb0a9dcbd41fbd4, R10, R11, R12, R13, R14, R15, R8, R9)
192 SHA512ROUND1(23, 0x76f988da831153b5, R9, R10, R11, R12, R13, R14, R15, R8)
193 SHA512ROUND1(24, 0x983e5152ee66dfab, R8, R9, R10, R11, R12, R13, R14, R15)
194 SHA512ROUND1(25, 0xa831c66d2db43210, R15, R8, R9, R10, R11, R12, R13, R14)
195 SHA512ROUND1(26, 0xb00327c898fb213f, R14, R15, R8, R9, R10, R11, R12, R13)
196 SHA512ROUND1(27, 0xbf597fc7beef0ee4, R13, R14, R15, R8, R9, R10, R11, R12)
197 SHA512ROUND1(28, 0xc6e00bf33da88fc2, R12, R13, R14, R15, R8, R9, R10, R11)
198 SHA512ROUND1(29, 0xd5a79147930aa725, R11, R12, R13, R14, R15, R8, R9, R10)
199 SHA512ROUND1(30, 0x06ca6351e003826f, R10, R11, R12, R13, R14, R15, R8, R9)
200 SHA512ROUND1(31, 0x142929670a0e6e70, R9, R10, R11, R12, R13, R14, R15, R8)
201 SHA512ROUND1(32, 0x27b70a8546d22ffc, R8, R9, R10, R11, R12, R13, R14, R15)
202 SHA512ROUND1(33, 0x2e1b21385c26c926, R15, R8, R9, R10, R11, R12, R13, R14)
203 SHA512ROUND1(34, 0x4d2c6dfc5ac42aed, R14, R15, R8, R9, R10, R11, R12, R13)
204 SHA512ROUND1(35, 0x53380d139d95b3df, R13, R14, R15, R8, R9, R10, R11, R12)
205 SHA512ROUND1(36, 0x650a73548baf63de, R12, R13, R14, R15, R8, R9, R10, R11)
206 SHA512ROUND1(37, 0x766a0abb3c77b2a8, R11, R12, R13, R14, R15, R8, R9, R10)
207 SHA512ROUND1(38, 0x81c2c92e47edaee6, R10, R11, R12, R13, R14, R15, R8, R9)
208 SHA512ROUND1(39, 0x92722c851482353b, R9, R10, R11, R12, R13, R14, R15, R8)
209 SHA512ROUND1(40, 0xa2bfe8a14cf10364, R8, R9, R10, R11, R12, R13, R14, R15)
210 SHA512ROUND1(41, 0xa81a664bbc423001, R15, R8, R9, R10, R11, R12, R13, R14)
211 SHA512ROUND1(42, 0xc24b8b70d0f89791, R14, R15, R8, R9, R10, R11, R12, R13)
212 SHA512ROUND1(43, 0xc76c51a30654be30, R13, R14, R15, R8, R9, R10, R11, R12)
213 SHA512ROUND1(44, 0xd192e819d6ef5218, R12, R13, R14, R15, R8, R9, R10, R11)
214 SHA512ROUND1(45, 0xd69906245565a910, R11, R12, R13, R14, R15, R8, R9, R10)
215 SHA512ROUND1(46, 0xf40e35855771202a, R10, R11, R12, R13, R14, R15, R8, R9)
216 SHA512ROUND1(47, 0x106aa07032bbd1b8, R9, R10, R11, R12, R13, R14, R15, R8)
217 SHA512ROUND1(48, 0x19a4c116b8d2d0c8, R8, R9, R10, R11, R12, R13, R14, R15)
218 SHA512ROUND1(49, 0x1e376c085141ab53, R15, R8, R9, R10, R11, R12, R13, R14)
219 SHA512ROUND1(50, 0x2748774cdf8eeb99, R14, R15, R8, R9, R10, R11, R12, R13)
220 SHA512ROUND1(51, 0x34b0bcb5e19b48a8, R13, R14, R15, R8, R9, R10, R11, R12)
221 SHA512ROUND1(52, 0x391c0cb3c5c95a63, R12, R13, R14, R15, R8, R9, R10, R11)
222 SHA512ROUND1(53, 0x4ed8aa4ae3418acb, R11, R12, R13, R14, R15, R8, R9, R10)
223 SHA512ROUND1(54, 0x5b9cca4f7763e373, R10, R11, R12, R13, R14, R15, R8, R9)
224 SHA512ROUND1(55, 0x682e6ff3d6b2b8a3, R9, R10, R11, R12, R13, R14, R15, R8)
225 SHA512ROUND1(56, 0x748f82ee5defb2fc, R8, R9, R10, R11, R12, R13, R14, R15)
226 SHA512ROUND1(57, 0x78a5636f43172f60, R15, R8, R9, R10, R11, R12, R13, R14)
227 SHA512ROUND1(58, 0x84c87814a1f0ab72, R14, R15, R8, R9, R10, R11, R12, R13)
228 SHA512ROUND1(59, 0x8cc702081a6439ec, R13, R14, R15, R8, R9, R10, R11, R12)
229 SHA512ROUND1(60, 0x90befffa23631e28, R12, R13, R14, R15, R8, R9, R10, R11)
230 SHA512ROUND1(61, 0xa4506cebde82bde9, R11, R12, R13, R14, R15, R8, R9, R10)
231 SHA512ROUND1(62, 0xbef9a3f7b2c67915, R10, R11, R12, R13, R14, R15, R8, R9)
232 SHA512ROUND1(63, 0xc67178f2e372532b, R9, R10, R11, R12, R13, R14, R15, R8)
233 SHA512ROUND1(64, 0xca273eceea26619c, R8, R9, R10, R11, R12, R13, R14, R15)
234 SHA512ROUND1(65, 0xd186b8c721c0c207, R15, R8, R9, R10, R11, R12, R13, R14)
235 SHA512ROUND1(66, 0xeada7dd6cde0eb1e, R14, R15, R8, R9, R10, R11, R12, R13)
236 SHA512ROUND1(67, 0xf57d4f7fee6ed178, R13, R14, R15, R8, R9, R10, R11, R12)
237 SHA512ROUND1(68, 0x06f067aa72176fba, R12, R13, R14, R15, R8, R9, R10, R11)
238 SHA512ROUND1(69, 0x0a637dc5a2c898a6, R11, R12, R13, R14, R15, R8, R9, R10)
239 SHA512ROUND1(70, 0x113f9804bef90dae, R10, R11, R12, R13, R14, R15, R8, R9)
240 SHA512ROUND1(71, 0x1b710b35131c471b, R9, R10, R11, R12, R13, R14, R15, R8)
241 SHA512ROUND1(72, 0x28db77f523047d84, R8, R9, R10, R11, R12, R13, R14, R15)
242 SHA512ROUND1(73, 0x32caab7b40c72493, R15, R8, R9, R10, R11, R12, R13, R14)
243 SHA512ROUND1(74, 0x3c9ebe0a15c9bebc, R14, R15, R8, R9, R10, R11, R12, R13)
244 SHA512ROUND1(75, 0x431d67c49c100d4c, R13, R14, R15, R8, R9, R10, R11, R12)
245 SHA512ROUND1(76, 0x4cc5d4becb3e42b6, R12, R13, R14, R15, R8, R9, R10, R11)
246 SHA512ROUND1(77, 0x597f299cfc657e2a, R11, R12, R13, R14, R15, R8, R9, R10)
247 SHA512ROUND1(78, 0x5fcb6fab3ad6faec, R10, R11, R12, R13, R14, R15, R8, R9)
248 SHA512ROUND1(79, 0x6c44198c4a475817, R9, R10, R11, R12, R13, R14, R15, R8)
249
250 MOVQ dig+0(FP), BP
251 ADDQ (0*8)(BP), R8 // H0 = a + H0
252 MOVQ R8, (0*8)(BP)
253 ADDQ (1*8)(BP), R9 // H1 = b + H1
254 MOVQ R9, (1*8)(BP)
255 ADDQ (2*8)(BP), R10 // H2 = c + H2
256 MOVQ R10, (2*8)(BP)
257 ADDQ (3*8)(BP), R11 // H3 = d + H3
258 MOVQ R11, (3*8)(BP)
259 ADDQ (4*8)(BP), R12 // H4 = e + H4
260 MOVQ R12, (4*8)(BP)
261 ADDQ (5*8)(BP), R13 // H5 = f + H5
262 MOVQ R13, (5*8)(BP)
263 ADDQ (6*8)(BP), R14 // H6 = g + H6
264 MOVQ R14, (6*8)(BP)
265 ADDQ (7*8)(BP), R15 // H7 = h + H7
266 MOVQ R15, (7*8)(BP)
267
268 ADDQ $128, SI
269 CMPQ SI, 640(SP)
270 JB loop
271
272 end:
273 RET
274
275 // Version below is based on "Fast SHA512 Implementations on Intel
276 // Architecture Processors" White-paper
277 // https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-sha512-implementations-ia-processors-paper.pdf
278 // AVX2 version by Intel, same algorithm in Linux kernel:
279 // https://github.com/torvalds/linux/blob/master/arch/x86/crypto/sha512-avx2-asm.S
280
281 // James Guilford <james.guilford@intel.com>
282 // Kirk Yap <kirk.s.yap@intel.com>
283 // Tim Chen <tim.c.chen@linux.intel.com>
284 // David Cote <david.m.cote@intel.com>
285 // Aleksey Sidorov <aleksey.sidorov@intel.com>
286
287 #define YFER_SIZE (4*8)
288 #define SRND_SIZE (1*8)
289 #define INP_SIZE (1*8)
290
291 #define frame_YFER (0)
292 #define frame_SRND (frame_YFER + YFER_SIZE)
293 #define frame_INP (frame_SRND + SRND_SIZE)
294 #define frame_INPEND (frame_INP + INP_SIZE)
295
296 #define addm(p1, p2) \
297 ADDQ p1, p2; \
298 MOVQ p2, p1
299
300 #define COPY_YMM_AND_BSWAP(p1, p2, p3) \
301 VMOVDQU p2, p1; \
302 VPSHUFB p3, p1, p1
303
304 #define MY_VPALIGNR(YDST, YSRC1, YSRC2, RVAL) \
305 VPERM2F128 $0x3, YSRC2, YSRC1, YDST; \
306 VPALIGNR $RVAL, YSRC2, YDST, YDST
307
308 DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x00(SB)/8, $0x0001020304050607
309 DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x08(SB)/8, $0x08090a0b0c0d0e0f
310 DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x10(SB)/8, $0x1011121314151617
311 DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x18(SB)/8, $0x18191a1b1c1d1e1f
312
313 GLOBL PSHUFFLE_BYTE_FLIP_MASK<>(SB), (NOPTR+RODATA), $32
314
315 DATA MASK_YMM_LO<>+0x00(SB)/8, $0x0000000000000000
316 DATA MASK_YMM_LO<>+0x08(SB)/8, $0x0000000000000000
317 DATA MASK_YMM_LO<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
318 DATA MASK_YMM_LO<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
319
320 GLOBL MASK_YMM_LO<>(SB), (NOPTR+RODATA), $32
321
322 TEXT ·blockAVX2(SB), NOSPLIT, $56-32
323 MOVQ dig+0(FP), SI
324 MOVQ p_base+8(FP), DI
325 MOVQ p_len+16(FP), DX
326
327 SHRQ $7, DX
328 SHLQ $7, DX
329
330 JZ done_hash
331 ADDQ DI, DX
332 MOVQ DX, frame_INPEND(SP)
333
334 MOVQ (0*8)(SI), AX
335 MOVQ (1*8)(SI), BX
336 MOVQ (2*8)(SI), CX
337 MOVQ (3*8)(SI), R8
338 MOVQ (4*8)(SI), DX
339 MOVQ (5*8)(SI), R9
340 MOVQ (6*8)(SI), R10
341 MOVQ (7*8)(SI), R11
342
343 VMOVDQU PSHUFFLE_BYTE_FLIP_MASK<>(SB), Y9
344
345 loop0:
346 MOVQ ·_K+0(SB), BP
347
348 // byte swap first 16 dwords
349 COPY_YMM_AND_BSWAP(Y4, (0*32)(DI), Y9)
350 COPY_YMM_AND_BSWAP(Y5, (1*32)(DI), Y9)
351 COPY_YMM_AND_BSWAP(Y6, (2*32)(DI), Y9)
352 COPY_YMM_AND_BSWAP(Y7, (3*32)(DI), Y9)
353
354 MOVQ DI, frame_INP(SP)
355
356 // schedule 64 input dwords, by doing 12 rounds of 4 each
357 MOVQ $4, frame_SRND(SP)
358
359 loop1:
360 VPADDQ (BP), Y4, Y0
361 VMOVDQU Y0, frame_YFER(SP)
362
363 MY_VPALIGNR(Y0, Y7, Y6, 8)
364
365 VPADDQ Y4, Y0, Y0
366
367 MY_VPALIGNR(Y1, Y5, Y4, 8)
368
369 VPSRLQ $1, Y1, Y2
370 VPSLLQ $(64-1), Y1, Y3
371 VPOR Y2, Y3, Y3
372
373 VPSRLQ $7, Y1, Y8
374
375 MOVQ AX, DI
376 RORXQ $41, DX, R13
377 RORXQ $18, DX, R14
378 ADDQ frame_YFER(SP), R11
379 ORQ CX, DI
380 MOVQ R9, R15
381 RORXQ $34, AX, R12
382
383 XORQ R14, R13
384 XORQ R10, R15
385 RORXQ $14, DX, R14
386
387 ANDQ DX, R15
388 XORQ R14, R13
389 RORXQ $39, AX, R14
390 ADDQ R11, R8
391
392 ANDQ BX, DI
393 XORQ R12, R14
394 RORXQ $28, AX, R12
395
396 XORQ R10, R15
397 XORQ R12, R14
398 MOVQ AX, R12
399 ANDQ CX, R12
400
401 ADDQ R13, R15
402 ORQ R12, DI
403 ADDQ R14, R11
404
405 ADDQ R15, R8
406
407 ADDQ R15, R11
408 ADDQ DI, R11
409
410 VPSRLQ $8, Y1, Y2
411 VPSLLQ $(64-8), Y1, Y1
412 VPOR Y2, Y1, Y1
413
414 VPXOR Y8, Y3, Y3
415 VPXOR Y1, Y3, Y1
416
417 VPADDQ Y1, Y0, Y0
418
419 VPERM2F128 $0x0, Y0, Y0, Y4
420
421 VPAND MASK_YMM_LO<>(SB), Y0, Y0
422
423 VPERM2F128 $0x11, Y7, Y7, Y2
424 VPSRLQ $6, Y2, Y8
425
426 MOVQ R11, DI
427 RORXQ $41, R8, R13
428 RORXQ $18, R8, R14
429 ADDQ 1*8+frame_YFER(SP), R10
430 ORQ BX, DI
431
432 MOVQ DX, R15
433 RORXQ $34, R11, R12
434 XORQ R14, R13
435 XORQ R9, R15
436
437 RORXQ $14, R8, R14
438 XORQ R14, R13
439 RORXQ $39, R11, R14
440 ANDQ R8, R15
441 ADDQ R10, CX
442
443 ANDQ AX, DI
444 XORQ R12, R14
445
446 RORXQ $28, R11, R12
447 XORQ R9, R15
448
449 XORQ R12, R14
450 MOVQ R11, R12
451 ANDQ BX, R12
452 ADDQ R13, R15
453
454 ORQ R12, DI
455 ADDQ R14, R10
456
457 ADDQ R15, CX
458 ADDQ R15, R10
459 ADDQ DI, R10
460
461 VPSRLQ $19, Y2, Y3
462 VPSLLQ $(64-19), Y2, Y1
463 VPOR Y1, Y3, Y3
464 VPXOR Y3, Y8, Y8
465 VPSRLQ $61, Y2, Y3
466 VPSLLQ $(64-61), Y2, Y1
467 VPOR Y1, Y3, Y3
468 VPXOR Y3, Y8, Y8
469
470 VPADDQ Y8, Y4, Y4
471
472 VPSRLQ $6, Y4, Y8
473
474 MOVQ R10, DI
475 RORXQ $41, CX, R13
476 ADDQ 2*8+frame_YFER(SP), R9
477
478 RORXQ $18, CX, R14
479 ORQ AX, DI
480 MOVQ R8, R15
481 XORQ DX, R15
482
483 RORXQ $34, R10, R12
484 XORQ R14, R13
485 ANDQ CX, R15
486
487 RORXQ $14, CX, R14
488 ADDQ R9, BX
489 ANDQ R11, DI
490
491 XORQ R14, R13
492 RORXQ $39, R10, R14
493 XORQ DX, R15
494
495 XORQ R12, R14
496 RORXQ $28, R10, R12
497
498 XORQ R12, R14
499 MOVQ R10, R12
500 ANDQ AX, R12
501 ADDQ R13, R15
502
503 ORQ R12, DI
504 ADDQ R14, R9
505 ADDQ R15, BX
506 ADDQ R15, R9
507
508 ADDQ DI, R9
509
510 VPSRLQ $19, Y4, Y3
511 VPSLLQ $(64-19), Y4, Y1
512 VPOR Y1, Y3, Y3
513 VPXOR Y3, Y8, Y8
514 VPSRLQ $61, Y4, Y3
515 VPSLLQ $(64-61), Y4, Y1
516 VPOR Y1, Y3, Y3
517 VPXOR Y3, Y8, Y8
518
519 VPADDQ Y8, Y0, Y2
520
521 VPBLENDD $0xF0, Y2, Y4, Y4
522
523 MOVQ R9, DI
524 RORXQ $41, BX, R13
525 RORXQ $18, BX, R14
526 ADDQ 3*8+frame_YFER(SP), DX
527 ORQ R11, DI
528
529 MOVQ CX, R15
530 RORXQ $34, R9, R12
531 XORQ R14, R13
532 XORQ R8, R15
533
534 RORXQ $14, BX, R14
535 ANDQ BX, R15
536 ADDQ DX, AX
537 ANDQ R10, DI
538
539 XORQ R14, R13
540 XORQ R8, R15
541
542 RORXQ $39, R9, R14
543 ADDQ R13, R15
544
545 XORQ R12, R14
546 ADDQ R15, AX
547
548 RORXQ $28, R9, R12
549
550 XORQ R12, R14
551 MOVQ R9, R12
552 ANDQ R11, R12
553 ORQ R12, DI
554
555 ADDQ R14, DX
556 ADDQ R15, DX
557 ADDQ DI, DX
558
559 VPADDQ 1*32(BP), Y5, Y0
560 VMOVDQU Y0, frame_YFER(SP)
561
562 MY_VPALIGNR(Y0, Y4, Y7, 8)
563
564 VPADDQ Y5, Y0, Y0
565
566 MY_VPALIGNR(Y1, Y6, Y5, 8)
567
568 VPSRLQ $1, Y1, Y2
569 VPSLLQ $(64-1), Y1, Y3
570 VPOR Y2, Y3, Y3
571
572 VPSRLQ $7, Y1, Y8
573
574 MOVQ DX, DI
575 RORXQ $41, AX, R13
576 RORXQ $18, AX, R14
577 ADDQ frame_YFER(SP), R8
578 ORQ R10, DI
579 MOVQ BX, R15
580 RORXQ $34, DX, R12
581
582 XORQ R14, R13
583 XORQ CX, R15
584 RORXQ $14, AX, R14
585
586 ANDQ AX, R15
587 XORQ R14, R13
588 RORXQ $39, DX, R14
589 ADDQ R8, R11
590
591 ANDQ R9, DI
592 XORQ R12, R14
593 RORXQ $28, DX, R12
594
595 XORQ CX, R15
596 XORQ R12, R14
597 MOVQ DX, R12
598 ANDQ R10, R12
599
600 ADDQ R13, R15
601 ORQ R12, DI
602 ADDQ R14, R8
603
604 ADDQ R15, R11
605
606 ADDQ R15, R8
607 ADDQ DI, R8
608
609 VPSRLQ $8, Y1, Y2
610 VPSLLQ $(64-8), Y1, Y1
611 VPOR Y2, Y1, Y1
612
613 VPXOR Y8, Y3, Y3
614 VPXOR Y1, Y3, Y1
615
616 VPADDQ Y1, Y0, Y0
617
618 VPERM2F128 $0x0, Y0, Y0, Y5
619
620 VPAND MASK_YMM_LO<>(SB), Y0, Y0
621
622 VPERM2F128 $0x11, Y4, Y4, Y2
623 VPSRLQ $6, Y2, Y8
624
625 MOVQ R8, DI
626 RORXQ $41, R11, R13
627 RORXQ $18, R11, R14
628 ADDQ 1*8+frame_YFER(SP), CX
629 ORQ R9, DI
630
631 MOVQ AX, R15
632 RORXQ $34, R8, R12
633 XORQ R14, R13
634 XORQ BX, R15
635
636 RORXQ $14, R11, R14
637 XORQ R14, R13
638 RORXQ $39, R8, R14
639 ANDQ R11, R15
640 ADDQ CX, R10
641
642 ANDQ DX, DI
643 XORQ R12, R14
644
645 RORXQ $28, R8, R12
646 XORQ BX, R15
647
648 XORQ R12, R14
649 MOVQ R8, R12
650 ANDQ R9, R12
651 ADDQ R13, R15
652
653 ORQ R12, DI
654 ADDQ R14, CX
655
656 ADDQ R15, R10
657 ADDQ R15, CX
658 ADDQ DI, CX
659
660 VPSRLQ $19, Y2, Y3
661 VPSLLQ $(64-19), Y2, Y1
662 VPOR Y1, Y3, Y3
663 VPXOR Y3, Y8, Y8
664 VPSRLQ $61, Y2, Y3
665 VPSLLQ $(64-61), Y2, Y1
666 VPOR Y1, Y3, Y3
667 VPXOR Y3, Y8, Y8
668
669 VPADDQ Y8, Y5, Y5
670
671 VPSRLQ $6, Y5, Y8
672
673 MOVQ CX, DI
674 RORXQ $41, R10, R13
675 ADDQ 2*8+frame_YFER(SP), BX
676
677 RORXQ $18, R10, R14
678 ORQ DX, DI
679 MOVQ R11, R15
680 XORQ AX, R15
681
682 RORXQ $34, CX, R12
683 XORQ R14, R13
684 ANDQ R10, R15
685
686 RORXQ $14, R10, R14
687 ADDQ BX, R9
688 ANDQ R8, DI
689
690 XORQ R14, R13
691 RORXQ $39, CX, R14
692 XORQ AX, R15
693
694 XORQ R12, R14
695 RORXQ $28, CX, R12
696
697 XORQ R12, R14
698 MOVQ CX, R12
699 ANDQ DX, R12
700 ADDQ R13, R15
701
702 ORQ R12, DI
703 ADDQ R14, BX
704 ADDQ R15, R9
705 ADDQ R15, BX
706
707 ADDQ DI, BX
708
709 VPSRLQ $19, Y5, Y3
710 VPSLLQ $(64-19), Y5, Y1
711 VPOR Y1, Y3, Y3
712 VPXOR Y3, Y8, Y8
713 VPSRLQ $61, Y5, Y3
714 VPSLLQ $(64-61), Y5, Y1
715 VPOR Y1, Y3, Y3
716 VPXOR Y3, Y8, Y8
717
718 VPADDQ Y8, Y0, Y2
719
720 VPBLENDD $0xF0, Y2, Y5, Y5
721
722 MOVQ BX, DI
723 RORXQ $41, R9, R13
724 RORXQ $18, R9, R14
725 ADDQ 3*8+frame_YFER(SP), AX
726 ORQ R8, DI
727
728 MOVQ R10, R15
729 RORXQ $34, BX, R12
730 XORQ R14, R13
731 XORQ R11, R15
732
733 RORXQ $14, R9, R14
734 ANDQ R9, R15
735 ADDQ AX, DX
736 ANDQ CX, DI
737
738 XORQ R14, R13
739 XORQ R11, R15
740
741 RORXQ $39, BX, R14
742 ADDQ R13, R15
743
744 XORQ R12, R14
745 ADDQ R15, DX
746
747 RORXQ $28, BX, R12
748
749 XORQ R12, R14
750 MOVQ BX, R12
751 ANDQ R8, R12
752 ORQ R12, DI
753
754 ADDQ R14, AX
755 ADDQ R15, AX
756 ADDQ DI, AX
757
758 VPADDQ 2*32(BP), Y6, Y0
759 VMOVDQU Y0, frame_YFER(SP)
760
761 MY_VPALIGNR(Y0, Y5, Y4, 8)
762
763 VPADDQ Y6, Y0, Y0
764
765 MY_VPALIGNR(Y1, Y7, Y6, 8)
766
767 VPSRLQ $1, Y1, Y2
768 VPSLLQ $(64-1), Y1, Y3
769 VPOR Y2, Y3, Y3
770
771 VPSRLQ $7, Y1, Y8
772
773 MOVQ AX, DI
774 RORXQ $41, DX, R13
775 RORXQ $18, DX, R14
776 ADDQ frame_YFER(SP), R11
777 ORQ CX, DI
778 MOVQ R9, R15
779 RORXQ $34, AX, R12
780
781 XORQ R14, R13
782 XORQ R10, R15
783 RORXQ $14, DX, R14
784
785 ANDQ DX, R15
786 XORQ R14, R13
787 RORXQ $39, AX, R14
788 ADDQ R11, R8
789
790 ANDQ BX, DI
791 XORQ R12, R14
792 RORXQ $28, AX, R12
793
794 XORQ R10, R15
795 XORQ R12, R14
796 MOVQ AX, R12
797 ANDQ CX, R12
798
799 ADDQ R13, R15
800 ORQ R12, DI
801 ADDQ R14, R11
802
803 ADDQ R15, R8
804
805 ADDQ R15, R11
806 ADDQ DI, R11
807
808 VPSRLQ $8, Y1, Y2
809 VPSLLQ $(64-8), Y1, Y1
810 VPOR Y2, Y1, Y1
811
812 VPXOR Y8, Y3, Y3
813 VPXOR Y1, Y3, Y1
814
815 VPADDQ Y1, Y0, Y0
816
817 VPERM2F128 $0x0, Y0, Y0, Y6
818
819 VPAND MASK_YMM_LO<>(SB), Y0, Y0
820
821 VPERM2F128 $0x11, Y5, Y5, Y2
822 VPSRLQ $6, Y2, Y8
823
824 MOVQ R11, DI
825 RORXQ $41, R8, R13
826 RORXQ $18, R8, R14
827 ADDQ 1*8+frame_YFER(SP), R10
828 ORQ BX, DI
829
830 MOVQ DX, R15
831 RORXQ $34, R11, R12
832 XORQ R14, R13
833 XORQ R9, R15
834
835 RORXQ $14, R8, R14
836 XORQ R14, R13
837 RORXQ $39, R11, R14
838 ANDQ R8, R15
839 ADDQ R10, CX
840
841 ANDQ AX, DI
842 XORQ R12, R14
843
844 RORXQ $28, R11, R12
845 XORQ R9, R15
846
847 XORQ R12, R14
848 MOVQ R11, R12
849 ANDQ BX, R12
850 ADDQ R13, R15
851
852 ORQ R12, DI
853 ADDQ R14, R10
854
855 ADDQ R15, CX
856 ADDQ R15, R10
857 ADDQ DI, R10
858
859 VPSRLQ $19, Y2, Y3
860 VPSLLQ $(64-19), Y2, Y1
861 VPOR Y1, Y3, Y3
862 VPXOR Y3, Y8, Y8
863 VPSRLQ $61, Y2, Y3
864 VPSLLQ $(64-61), Y2, Y1
865 VPOR Y1, Y3, Y3
866 VPXOR Y3, Y8, Y8
867
868 VPADDQ Y8, Y6, Y6
869
870 VPSRLQ $6, Y6, Y8
871
872 MOVQ R10, DI
873 RORXQ $41, CX, R13
874 ADDQ 2*8+frame_YFER(SP), R9
875
876 RORXQ $18, CX, R14
877 ORQ AX, DI
878 MOVQ R8, R15
879 XORQ DX, R15
880
881 RORXQ $34, R10, R12
882 XORQ R14, R13
883 ANDQ CX, R15
884
885 RORXQ $14, CX, R14
886 ADDQ R9, BX
887 ANDQ R11, DI
888
889 XORQ R14, R13
890 RORXQ $39, R10, R14
891 XORQ DX, R15
892
893 XORQ R12, R14
894 RORXQ $28, R10, R12
895
896 XORQ R12, R14
897 MOVQ R10, R12
898 ANDQ AX, R12
899 ADDQ R13, R15
900
901 ORQ R12, DI
902 ADDQ R14, R9
903 ADDQ R15, BX
904 ADDQ R15, R9
905
906 ADDQ DI, R9
907
908 VPSRLQ $19, Y6, Y3
909 VPSLLQ $(64-19), Y6, Y1
910 VPOR Y1, Y3, Y3
911 VPXOR Y3, Y8, Y8
912 VPSRLQ $61, Y6, Y3
913 VPSLLQ $(64-61), Y6, Y1
914 VPOR Y1, Y3, Y3
915 VPXOR Y3, Y8, Y8
916
917 VPADDQ Y8, Y0, Y2
918
919 VPBLENDD $0xF0, Y2, Y6, Y6
920
921 MOVQ R9, DI
922 RORXQ $41, BX, R13
923 RORXQ $18, BX, R14
924 ADDQ 3*8+frame_YFER(SP), DX
925 ORQ R11, DI
926
927 MOVQ CX, R15
928 RORXQ $34, R9, R12
929 XORQ R14, R13
930 XORQ R8, R15
931
932 RORXQ $14, BX, R14
933 ANDQ BX, R15
934 ADDQ DX, AX
935 ANDQ R10, DI
936
937 XORQ R14, R13
938 XORQ R8, R15
939
940 RORXQ $39, R9, R14
941 ADDQ R13, R15
942
943 XORQ R12, R14
944 ADDQ R15, AX
945
946 RORXQ $28, R9, R12
947
948 XORQ R12, R14
949 MOVQ R9, R12
950 ANDQ R11, R12
951 ORQ R12, DI
952
953 ADDQ R14, DX
954 ADDQ R15, DX
955 ADDQ DI, DX
956
957 VPADDQ 3*32(BP), Y7, Y0
958 VMOVDQU Y0, frame_YFER(SP)
959 ADDQ $(4*32), BP
960
961 MY_VPALIGNR(Y0, Y6, Y5, 8)
962
963 VPADDQ Y7, Y0, Y0
964
965 MY_VPALIGNR(Y1, Y4, Y7, 8)
966
967 VPSRLQ $1, Y1, Y2
968 VPSLLQ $(64-1), Y1, Y3
969 VPOR Y2, Y3, Y3
970
971 VPSRLQ $7, Y1, Y8
972
973 MOVQ DX, DI
974 RORXQ $41, AX, R13
975 RORXQ $18, AX, R14
976 ADDQ frame_YFER(SP), R8
977 ORQ R10, DI
978 MOVQ BX, R15
979 RORXQ $34, DX, R12
980
981 XORQ R14, R13
982 XORQ CX, R15
983 RORXQ $14, AX, R14
984
985 ANDQ AX, R15
986 XORQ R14, R13
987 RORXQ $39, DX, R14
988 ADDQ R8, R11
989
990 ANDQ R9, DI
991 XORQ R12, R14
992 RORXQ $28, DX, R12
993
994 XORQ CX, R15
995 XORQ R12, R14
996 MOVQ DX, R12
997 ANDQ R10, R12
998
999 ADDQ R13, R15
1000 ORQ R12, DI
1001 ADDQ R14, R8
1002
1003 ADDQ R15, R11
1004
1005 ADDQ R15, R8
1006 ADDQ DI, R8
1007
1008 VPSRLQ $8, Y1, Y2
1009 VPSLLQ $(64-8), Y1, Y1
1010 VPOR Y2, Y1, Y1
1011
1012 VPXOR Y8, Y3, Y3
1013 VPXOR Y1, Y3, Y1
1014
1015 VPADDQ Y1, Y0, Y0
1016
1017 VPERM2F128 $0x0, Y0, Y0, Y7
1018
1019 VPAND MASK_YMM_LO<>(SB), Y0, Y0
1020
1021 VPERM2F128 $0x11, Y6, Y6, Y2
1022 VPSRLQ $6, Y2, Y8
1023
1024 MOVQ R8, DI
1025 RORXQ $41, R11, R13
1026 RORXQ $18, R11, R14
1027 ADDQ 1*8+frame_YFER(SP), CX
1028 ORQ R9, DI
1029
1030 MOVQ AX, R15
1031 RORXQ $34, R8, R12
1032 XORQ R14, R13
1033 XORQ BX, R15
1034
1035 RORXQ $14, R11, R14
1036 XORQ R14, R13
1037 RORXQ $39, R8, R14
1038 ANDQ R11, R15
1039 ADDQ CX, R10
1040
1041 ANDQ DX, DI
1042 XORQ R12, R14
1043
1044 RORXQ $28, R8, R12
1045 XORQ BX, R15
1046
1047 XORQ R12, R14
1048 MOVQ R8, R12
1049 ANDQ R9, R12
1050 ADDQ R13, R15
1051
1052 ORQ R12, DI
1053 ADDQ R14, CX
1054
1055 ADDQ R15, R10
1056 ADDQ R15, CX
1057 ADDQ DI, CX
1058
1059 VPSRLQ $19, Y2, Y3
1060 VPSLLQ $(64-19), Y2, Y1
1061 VPOR Y1, Y3, Y3
1062 VPXOR Y3, Y8, Y8
1063 VPSRLQ $61, Y2, Y3
1064 VPSLLQ $(64-61), Y2, Y1
1065 VPOR Y1, Y3, Y3
1066 VPXOR Y3, Y8, Y8
1067
1068 VPADDQ Y8, Y7, Y7
1069
1070 VPSRLQ $6, Y7, Y8
1071
1072 MOVQ CX, DI
1073 RORXQ $41, R10, R13
1074 ADDQ 2*8+frame_YFER(SP), BX
1075
1076 RORXQ $18, R10, R14
1077 ORQ DX, DI
1078 MOVQ R11, R15
1079 XORQ AX, R15
1080
1081 RORXQ $34, CX, R12
1082 XORQ R14, R13
1083 ANDQ R10, R15
1084
1085 RORXQ $14, R10, R14
1086 ADDQ BX, R9
1087 ANDQ R8, DI
1088
1089 XORQ R14, R13
1090 RORXQ $39, CX, R14
1091 XORQ AX, R15
1092
1093 XORQ R12, R14
1094 RORXQ $28, CX, R12
1095
1096 XORQ R12, R14
1097 MOVQ CX, R12
1098 ANDQ DX, R12
1099 ADDQ R13, R15
1100
1101 ORQ R12, DI
1102 ADDQ R14, BX
1103 ADDQ R15, R9
1104 ADDQ R15, BX
1105
1106 ADDQ DI, BX
1107
1108 VPSRLQ $19, Y7, Y3
1109 VPSLLQ $(64-19), Y7, Y1
1110 VPOR Y1, Y3, Y3
1111 VPXOR Y3, Y8, Y8
1112 VPSRLQ $61, Y7, Y3
1113 VPSLLQ $(64-61), Y7, Y1
1114 VPOR Y1, Y3, Y3
1115 VPXOR Y3, Y8, Y8
1116
1117 VPADDQ Y8, Y0, Y2
1118
1119 VPBLENDD $0xF0, Y2, Y7, Y7
1120
1121 MOVQ BX, DI
1122 RORXQ $41, R9, R13
1123 RORXQ $18, R9, R14
1124 ADDQ 3*8+frame_YFER(SP), AX
1125 ORQ R8, DI
1126
1127 MOVQ R10, R15
1128 RORXQ $34, BX, R12
1129 XORQ R14, R13
1130 XORQ R11, R15
1131
1132 RORXQ $14, R9, R14
1133 ANDQ R9, R15
1134 ADDQ AX, DX
1135 ANDQ CX, DI
1136
1137 XORQ R14, R13
1138 XORQ R11, R15
1139
1140 RORXQ $39, BX, R14
1141 ADDQ R13, R15
1142
1143 XORQ R12, R14
1144 ADDQ R15, DX
1145
1146 RORXQ $28, BX, R12
1147
1148 XORQ R12, R14
1149 MOVQ BX, R12
1150 ANDQ R8, R12
1151 ORQ R12, DI
1152
1153 ADDQ R14, AX
1154 ADDQ R15, AX
1155 ADDQ DI, AX
1156
1157 SUBQ $1, frame_SRND(SP)
1158 JNE loop1
1159
1160 MOVQ $2, frame_SRND(SP)
1161
1162 loop2:
1163 VPADDQ (BP), Y4, Y0
1164 VMOVDQU Y0, frame_YFER(SP)
1165
1166 MOVQ R9, R15
1167 RORXQ $41, DX, R13
1168 RORXQ $18, DX, R14
1169 XORQ R10, R15
1170
1171 XORQ R14, R13
1172 RORXQ $14, DX, R14
1173 ANDQ DX, R15
1174
1175 XORQ R14, R13
1176 RORXQ $34, AX, R12
1177 XORQ R10, R15
1178 RORXQ $39, AX, R14
1179 MOVQ AX, DI
1180
1181 XORQ R12, R14
1182 RORXQ $28, AX, R12
1183 ADDQ frame_YFER(SP), R11
1184 ORQ CX, DI
1185
1186 XORQ R12, R14
1187 MOVQ AX, R12
1188 ANDQ BX, DI
1189 ANDQ CX, R12
1190 ADDQ R13, R15
1191
1192 ADDQ R11, R8
1193 ORQ R12, DI
1194 ADDQ R14, R11
1195
1196 ADDQ R15, R8
1197
1198 ADDQ R15, R11
1199 MOVQ DX, R15
1200 RORXQ $41, R8, R13
1201 RORXQ $18, R8, R14
1202 XORQ R9, R15
1203
1204 XORQ R14, R13
1205 RORXQ $14, R8, R14
1206 ANDQ R8, R15
1207 ADDQ DI, R11
1208
1209 XORQ R14, R13
1210 RORXQ $34, R11, R12
1211 XORQ R9, R15
1212 RORXQ $39, R11, R14
1213 MOVQ R11, DI
1214
1215 XORQ R12, R14
1216 RORXQ $28, R11, R12
1217 ADDQ 8*1+frame_YFER(SP), R10
1218 ORQ BX, DI
1219
1220 XORQ R12, R14
1221 MOVQ R11, R12
1222 ANDQ AX, DI
1223 ANDQ BX, R12
1224 ADDQ R13, R15
1225
1226 ADDQ R10, CX
1227 ORQ R12, DI
1228 ADDQ R14, R10
1229
1230 ADDQ R15, CX
1231
1232 ADDQ R15, R10
1233 MOVQ R8, R15
1234 RORXQ $41, CX, R13
1235 RORXQ $18, CX, R14
1236 XORQ DX, R15
1237
1238 XORQ R14, R13
1239 RORXQ $14, CX, R14
1240 ANDQ CX, R15
1241 ADDQ DI, R10
1242
1243 XORQ R14, R13
1244 RORXQ $34, R10, R12
1245 XORQ DX, R15
1246 RORXQ $39, R10, R14
1247 MOVQ R10, DI
1248
1249 XORQ R12, R14
1250 RORXQ $28, R10, R12
1251 ADDQ 8*2+frame_YFER(SP), R9
1252 ORQ AX, DI
1253
1254 XORQ R12, R14
1255 MOVQ R10, R12
1256 ANDQ R11, DI
1257 ANDQ AX, R12
1258 ADDQ R13, R15
1259
1260 ADDQ R9, BX
1261 ORQ R12, DI
1262 ADDQ R14, R9
1263
1264 ADDQ R15, BX
1265
1266 ADDQ R15, R9
1267 MOVQ CX, R15
1268 RORXQ $41, BX, R13
1269 RORXQ $18, BX, R14
1270 XORQ R8, R15
1271
1272 XORQ R14, R13
1273 RORXQ $14, BX, R14
1274 ANDQ BX, R15
1275 ADDQ DI, R9
1276
1277 XORQ R14, R13
1278 RORXQ $34, R9, R12
1279 XORQ R8, R15
1280 RORXQ $39, R9, R14
1281 MOVQ R9, DI
1282
1283 XORQ R12, R14
1284 RORXQ $28, R9, R12
1285 ADDQ 8*3+frame_YFER(SP), DX
1286 ORQ R11, DI
1287
1288 XORQ R12, R14
1289 MOVQ R9, R12
1290 ANDQ R10, DI
1291 ANDQ R11, R12
1292 ADDQ R13, R15
1293
1294 ADDQ DX, AX
1295 ORQ R12, DI
1296 ADDQ R14, DX
1297
1298 ADDQ R15, AX
1299
1300 ADDQ R15, DX
1301
1302 ADDQ DI, DX
1303
1304 VPADDQ 1*32(BP), Y5, Y0
1305 VMOVDQU Y0, frame_YFER(SP)
1306 ADDQ $(2*32), BP
1307
1308 MOVQ BX, R15
1309 RORXQ $41, AX, R13
1310 RORXQ $18, AX, R14
1311 XORQ CX, R15
1312
1313 XORQ R14, R13
1314 RORXQ $14, AX, R14
1315 ANDQ AX, R15
1316
1317 XORQ R14, R13
1318 RORXQ $34, DX, R12
1319 XORQ CX, R15
1320 RORXQ $39, DX, R14
1321 MOVQ DX, DI
1322
1323 XORQ R12, R14
1324 RORXQ $28, DX, R12
1325 ADDQ frame_YFER(SP), R8
1326 ORQ R10, DI
1327
1328 XORQ R12, R14
1329 MOVQ DX, R12
1330 ANDQ R9, DI
1331 ANDQ R10, R12
1332 ADDQ R13, R15
1333
1334 ADDQ R8, R11
1335 ORQ R12, DI
1336 ADDQ R14, R8
1337
1338 ADDQ R15, R11
1339
1340 ADDQ R15, R8
1341 MOVQ AX, R15
1342 RORXQ $41, R11, R13
1343 RORXQ $18, R11, R14
1344 XORQ BX, R15
1345
1346 XORQ R14, R13
1347 RORXQ $14, R11, R14
1348 ANDQ R11, R15
1349 ADDQ DI, R8
1350
1351 XORQ R14, R13
1352 RORXQ $34, R8, R12
1353 XORQ BX, R15
1354 RORXQ $39, R8, R14
1355 MOVQ R8, DI
1356
1357 XORQ R12, R14
1358 RORXQ $28, R8, R12
1359 ADDQ 8*1+frame_YFER(SP), CX
1360 ORQ R9, DI
1361
1362 XORQ R12, R14
1363 MOVQ R8, R12
1364 ANDQ DX, DI
1365 ANDQ R9, R12
1366 ADDQ R13, R15
1367
1368 ADDQ CX, R10
1369 ORQ R12, DI
1370 ADDQ R14, CX
1371
1372 ADDQ R15, R10
1373
1374 ADDQ R15, CX
1375 MOVQ R11, R15
1376 RORXQ $41, R10, R13
1377 RORXQ $18, R10, R14
1378 XORQ AX, R15
1379
1380 XORQ R14, R13
1381 RORXQ $14, R10, R14
1382 ANDQ R10, R15
1383 ADDQ DI, CX
1384
1385 XORQ R14, R13
1386 RORXQ $34, CX, R12
1387 XORQ AX, R15
1388 RORXQ $39, CX, R14
1389 MOVQ CX, DI
1390
1391 XORQ R12, R14
1392 RORXQ $28, CX, R12
1393 ADDQ 8*2+frame_YFER(SP), BX
1394 ORQ DX, DI
1395
1396 XORQ R12, R14
1397 MOVQ CX, R12
1398 ANDQ R8, DI
1399 ANDQ DX, R12
1400 ADDQ R13, R15
1401
1402 ADDQ BX, R9
1403 ORQ R12, DI
1404 ADDQ R14, BX
1405
1406 ADDQ R15, R9
1407
1408 ADDQ R15, BX
1409 MOVQ R10, R15
1410 RORXQ $41, R9, R13
1411 RORXQ $18, R9, R14
1412 XORQ R11, R15
1413
1414 XORQ R14, R13
1415 RORXQ $14, R9, R14
1416 ANDQ R9, R15
1417 ADDQ DI, BX
1418
1419 XORQ R14, R13
1420 RORXQ $34, BX, R12
1421 XORQ R11, R15
1422 RORXQ $39, BX, R14
1423 MOVQ BX, DI
1424
1425 XORQ R12, R14
1426 RORXQ $28, BX, R12
1427 ADDQ 8*3+frame_YFER(SP), AX
1428 ORQ R8, DI
1429
1430 XORQ R12, R14
1431 MOVQ BX, R12
1432 ANDQ CX, DI
1433 ANDQ R8, R12
1434 ADDQ R13, R15
1435
1436 ADDQ AX, DX
1437 ORQ R12, DI
1438 ADDQ R14, AX
1439
1440 ADDQ R15, DX
1441
1442 ADDQ R15, AX
1443
1444 ADDQ DI, AX
1445
1446 VMOVDQU Y6, Y4
1447 VMOVDQU Y7, Y5
1448
1449 SUBQ $1, frame_SRND(SP)
1450 JNE loop2
1451
1452 addm(8*0(SI),AX)
1453 addm(8*1(SI),BX)
1454 addm(8*2(SI),CX)
1455 addm(8*3(SI),R8)
1456 addm(8*4(SI),DX)
1457 addm(8*5(SI),R9)
1458 addm(8*6(SI),R10)
1459 addm(8*7(SI),R11)
1460
1461 MOVQ frame_INP(SP), DI
1462 ADDQ $128, DI
1463 CMPQ DI, frame_INPEND(SP)
1464 JNE loop0
1465
1466 done_hash:
1467 VZEROUPPER
1468 RET
View as plain text