charon-nm: Set DPD/close action to restart and enable indefinite keying tries
[strongswan.git] / src / libstrongswan / plugins / aesni / aesni_gcm.c
1 /*
2 * Copyright (C) 2015 Martin Willi
3 * Copyright (C) 2015 revosec AG
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version. See <http://www.fsf.org/copyleft/gpl.txt>.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 */
15
16 #include "aesni_gcm.h"
17 #include "aesni_key.h"
18
19 #include <crypto/iv/iv_gen_seq.h>
20
21 #include <tmmintrin.h>
22
23 #define NONCE_SIZE 12
24 #define IV_SIZE 8
25 #define SALT_SIZE (NONCE_SIZE - IV_SIZE)
26
27 /**
28 * Parallel pipelining
29 */
30 #define GCM_CRYPT_PARALLELISM 4
31
32 typedef struct private_aesni_gcm_t private_aesni_gcm_t;
33
34 /**
35 * GCM en/decryption method type
36 */
37 typedef void (*aesni_gcm_fn_t)(private_aesni_gcm_t*, size_t, u_char*, u_char*,
38 u_char*, size_t, u_char*, u_char*);
39
40 /**
41 * Private data of an aesni_gcm_t object.
42 */
43 struct private_aesni_gcm_t {
44
45 /**
46 * Public aesni_gcm_t interface.
47 */
48 aesni_gcm_t public;
49
50 /**
51 * Encryption key schedule
52 */
53 aesni_key_t *key;
54
55 /**
56 * IV generator.
57 */
58 iv_gen_t *iv_gen;
59
60 /**
61 * Length of the integrity check value
62 */
63 size_t icv_size;
64
65 /**
66 * Length of the key in bytes
67 */
68 size_t key_size;
69
70 /**
71 * GCM encryption function
72 */
73 aesni_gcm_fn_t encrypt;
74
75 /**
76 * GCM decryption function
77 */
78 aesni_gcm_fn_t decrypt;
79
80 /**
81 * salt to add to nonce
82 */
83 u_char salt[SALT_SIZE];
84
85 /**
86 * GHASH subkey H, big-endian
87 */
88 __m128i h;
89
90 /**
91 * GHASH key H^2, big-endian
92 */
93 __m128i hh;
94
95 /**
96 * GHASH key H^3, big-endian
97 */
98 __m128i hhh;
99
100 /**
101 * GHASH key H^4, big-endian
102 */
103 __m128i hhhh;
104 };
105
106 /**
107 * Byte-swap a 128-bit integer
108 */
109 static inline __m128i swap128(__m128i x)
110 {
111 return _mm_shuffle_epi8(x,
112 _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
113 }
114
115 /**
116 * Multiply two blocks in GF128
117 */
118 static __m128i mult_block(__m128i h, __m128i y)
119 {
120 __m128i t1, t2, t3, t4, t5, t6;
121
122 y = swap128(y);
123
124 t1 = _mm_clmulepi64_si128(h, y, 0x00);
125 t2 = _mm_clmulepi64_si128(h, y, 0x01);
126 t3 = _mm_clmulepi64_si128(h, y, 0x10);
127 t4 = _mm_clmulepi64_si128(h, y, 0x11);
128
129 t2 = _mm_xor_si128(t2, t3);
130 t3 = _mm_slli_si128(t2, 8);
131 t2 = _mm_srli_si128(t2, 8);
132 t1 = _mm_xor_si128(t1, t3);
133 t4 = _mm_xor_si128(t4, t2);
134
135 t5 = _mm_srli_epi32(t1, 31);
136 t1 = _mm_slli_epi32(t1, 1);
137 t6 = _mm_srli_epi32(t4, 31);
138 t4 = _mm_slli_epi32(t4, 1);
139
140 t3 = _mm_srli_si128(t5, 12);
141 t6 = _mm_slli_si128(t6, 4);
142 t5 = _mm_slli_si128(t5, 4);
143 t1 = _mm_or_si128(t1, t5);
144 t4 = _mm_or_si128(t4, t6);
145 t4 = _mm_or_si128(t4, t3);
146
147 t5 = _mm_slli_epi32(t1, 31);
148 t6 = _mm_slli_epi32(t1, 30);
149 t3 = _mm_slli_epi32(t1, 25);
150
151 t5 = _mm_xor_si128(t5, t6);
152 t5 = _mm_xor_si128(t5, t3);
153 t6 = _mm_srli_si128(t5, 4);
154 t4 = _mm_xor_si128(t4, t6);
155 t5 = _mm_slli_si128(t5, 12);
156 t1 = _mm_xor_si128(t1, t5);
157 t4 = _mm_xor_si128(t4, t1);
158
159 t5 = _mm_srli_epi32(t1, 1);
160 t2 = _mm_srli_epi32(t1, 2);
161 t3 = _mm_srli_epi32(t1, 7);
162 t4 = _mm_xor_si128(t4, t2);
163 t4 = _mm_xor_si128(t4, t3);
164 t4 = _mm_xor_si128(t4, t5);
165
166 return swap128(t4);
167 }
168
169 /**
170 * Multiply four consecutive blocks by their respective GHASH key, XOR
171 */
172 static inline __m128i mult4xor(__m128i h1, __m128i h2, __m128i h3, __m128i h4,
173 __m128i d1, __m128i d2, __m128i d3, __m128i d4)
174 {
175 __m128i t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
176
177 d1 = swap128(d1);
178 d2 = swap128(d2);
179 d3 = swap128(d3);
180 d4 = swap128(d4);
181
182 t0 = _mm_clmulepi64_si128(h1, d1, 0x00);
183 t1 = _mm_clmulepi64_si128(h2, d2, 0x00);
184 t2 = _mm_clmulepi64_si128(h3, d3, 0x00);
185 t3 = _mm_clmulepi64_si128(h4, d4, 0x00);
186 t8 = _mm_xor_si128(t0, t1);
187 t8 = _mm_xor_si128(t8, t2);
188 t8 = _mm_xor_si128(t8, t3);
189
190 t4 = _mm_clmulepi64_si128(h1, d1, 0x11);
191 t5 = _mm_clmulepi64_si128(h2, d2, 0x11);
192 t6 = _mm_clmulepi64_si128(h3, d3, 0x11);
193 t7 = _mm_clmulepi64_si128(h4, d4, 0x11);
194 t9 = _mm_xor_si128(t4, t5);
195 t9 = _mm_xor_si128(t9, t6);
196 t9 = _mm_xor_si128(t9, t7);
197
198 t0 = _mm_shuffle_epi32(h1, 78);
199 t4 = _mm_shuffle_epi32(d1, 78);
200 t0 = _mm_xor_si128(t0, h1);
201 t4 = _mm_xor_si128(t4, d1);
202 t1 = _mm_shuffle_epi32(h2, 78);
203 t5 = _mm_shuffle_epi32(d2, 78);
204 t1 = _mm_xor_si128(t1, h2);
205 t5 = _mm_xor_si128(t5, d2);
206 t2 = _mm_shuffle_epi32(h3, 78);
207 t6 = _mm_shuffle_epi32(d3, 78);
208 t2 = _mm_xor_si128(t2, h3);
209 t6 = _mm_xor_si128(t6, d3);
210 t3 = _mm_shuffle_epi32(h4, 78);
211 t7 = _mm_shuffle_epi32(d4, 78);
212 t3 = _mm_xor_si128(t3, h4);
213 t7 = _mm_xor_si128(t7, d4);
214
215 t0 = _mm_clmulepi64_si128(t0, t4, 0x00);
216 t1 = _mm_clmulepi64_si128(t1, t5, 0x00);
217 t2 = _mm_clmulepi64_si128(t2, t6, 0x00);
218 t3 = _mm_clmulepi64_si128(t3, t7, 0x00);
219 t0 = _mm_xor_si128(t0, t8);
220 t0 = _mm_xor_si128(t0, t9);
221 t0 = _mm_xor_si128(t1, t0);
222 t0 = _mm_xor_si128(t2, t0);
223
224 t0 = _mm_xor_si128(t3, t0);
225 t4 = _mm_slli_si128(t0, 8);
226 t0 = _mm_srli_si128(t0, 8);
227 t3 = _mm_xor_si128(t4, t8);
228 t6 = _mm_xor_si128(t0, t9);
229 t7 = _mm_srli_epi32(t3, 31);
230 t8 = _mm_srli_epi32(t6, 31);
231 t3 = _mm_slli_epi32(t3, 1);
232 t6 = _mm_slli_epi32(t6, 1);
233 t9 = _mm_srli_si128(t7, 12);
234 t8 = _mm_slli_si128(t8, 4);
235 t7 = _mm_slli_si128(t7, 4);
236 t3 = _mm_or_si128(t3, t7);
237 t6 = _mm_or_si128(t6, t8);
238 t6 = _mm_or_si128(t6, t9);
239 t7 = _mm_slli_epi32(t3, 31);
240 t8 = _mm_slli_epi32(t3, 30);
241 t9 = _mm_slli_epi32(t3, 25);
242 t7 = _mm_xor_si128(t7, t8);
243 t7 = _mm_xor_si128(t7, t9);
244 t8 = _mm_srli_si128(t7, 4);
245 t7 = _mm_slli_si128(t7, 12);
246 t3 = _mm_xor_si128(t3, t7);
247 t2 = _mm_srli_epi32(t3, 1);
248 t4 = _mm_srli_epi32(t3, 2);
249 t5 = _mm_srli_epi32(t3, 7);
250 t2 = _mm_xor_si128(t2, t4);
251 t2 = _mm_xor_si128(t2, t5);
252 t2 = _mm_xor_si128(t2, t8);
253 t3 = _mm_xor_si128(t3, t2);
254 t6 = _mm_xor_si128(t6, t3);
255
256 return swap128(t6);
257 }
258
259 /**
260 * GHASH on a single block
261 */
262 static __m128i ghash(__m128i h, __m128i y, __m128i x)
263 {
264 return mult_block(h, _mm_xor_si128(y, x));
265 }
266
267 /**
268 * Start constructing the ICV for the associated data
269 */
270 static __m128i icv_header(private_aesni_gcm_t *this, void *assoc, size_t alen)
271 {
272 u_int blocks, pblocks, rem, i;
273 __m128i h1, h2, h3, h4, d1, d2, d3, d4;
274 __m128i y, last, *ab;
275
276 h1 = this->hhhh;
277 h2 = this->hhh;
278 h3 = this->hh;
279 h4 = this->h;
280
281 y = _mm_setzero_si128();
282 ab = assoc;
283 blocks = alen / AES_BLOCK_SIZE;
284 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
285 rem = alen % AES_BLOCK_SIZE;
286 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
287 {
288 d1 = _mm_loadu_si128(ab + i + 0);
289 d2 = _mm_loadu_si128(ab + i + 1);
290 d3 = _mm_loadu_si128(ab + i + 2);
291 d4 = _mm_loadu_si128(ab + i + 3);
292 y = _mm_xor_si128(y, d1);
293 y = mult4xor(h1, h2, h3, h4, y, d2, d3, d4);
294 }
295 for (i = pblocks; i < blocks; i++)
296 {
297 y = ghash(this->h, y, _mm_loadu_si128(ab + i));
298 }
299 if (rem)
300 {
301 last = _mm_setzero_si128();
302 memcpy(&last, ab + blocks, rem);
303
304 y = ghash(this->h, y, last);
305 }
306
307 return y;
308 }
309
310 /**
311 * Complete the ICV by hashing a assoc/data length block
312 */
313 static __m128i icv_tailer(private_aesni_gcm_t *this, __m128i y,
314 size_t alen, size_t dlen)
315 {
316 __m128i b;
317
318 htoun64(&b, alen * 8);
319 htoun64((u_char*)&b + sizeof(uint64_t), dlen * 8);
320
321 return ghash(this->h, y, b);
322 }
323
324 /**
325 * En-/Decrypt the ICV, trim and store it
326 */
327 static void icv_crypt(private_aesni_gcm_t *this, __m128i y, __m128i j,
328 u_char *icv)
329 {
330 __m128i *ks, t, b;
331 u_int round;
332
333 ks = this->key->schedule;
334 t = _mm_xor_si128(j, ks[0]);
335 for (round = 1; round < this->key->rounds; round++)
336 {
337 t = _mm_aesenc_si128(t, ks[round]);
338 }
339 t = _mm_aesenclast_si128(t, ks[this->key->rounds]);
340
341 t = _mm_xor_si128(y, t);
342
343 _mm_storeu_si128(&b, t);
344 memcpy(icv, &b, this->icv_size);
345 }
346
347 /**
348 * Do big-endian increment on x
349 */
350 static inline __m128i increment_be(__m128i x)
351 {
352 x = swap128(x);
353 x = _mm_add_epi64(x, _mm_set_epi32(0, 0, 0, 1));
354 x = swap128(x);
355
356 return x;
357 }
358
359 /**
360 * Generate the block J0
361 */
362 static inline __m128i create_j(private_aesni_gcm_t *this, u_char *iv)
363 {
364 u_char j[AES_BLOCK_SIZE];
365
366 memcpy(j, this->salt, SALT_SIZE);
367 memcpy(j + SALT_SIZE, iv, IV_SIZE);
368 htoun32(j + SALT_SIZE + IV_SIZE, 1);
369
370 return _mm_loadu_si128((__m128i*)j);
371 }
372
373 /**
374 * Encrypt a remaining incomplete block, return updated Y
375 */
376 static __m128i encrypt_gcm_rem(private_aesni_gcm_t *this, u_int rem,
377 void *in, void *out, __m128i cb, __m128i y)
378 {
379 __m128i *ks, t, b;
380 u_int round;
381
382 memset(&b, 0, sizeof(b));
383 memcpy(&b, in, rem);
384
385 ks = this->key->schedule;
386 t = _mm_xor_si128(cb, ks[0]);
387 for (round = 1; round < this->key->rounds; round++)
388 {
389 t = _mm_aesenc_si128(t, ks[round]);
390 }
391 t = _mm_aesenclast_si128(t, ks[this->key->rounds]);
392 b = _mm_xor_si128(t, b);
393
394 memcpy(out, &b, rem);
395
396 memset((u_char*)&b + rem, 0, AES_BLOCK_SIZE - rem);
397 return ghash(this->h, y, b);
398 }
399
400 /**
401 * Decrypt a remaining incomplete block, return updated Y
402 */
403 static __m128i decrypt_gcm_rem(private_aesni_gcm_t *this, u_int rem,
404 void *in, void *out, __m128i cb, __m128i y)
405 {
406 __m128i *ks, t, b;
407 u_int round;
408
409 memset(&b, 0, sizeof(b));
410 memcpy(&b, in, rem);
411
412 y = ghash(this->h, y, b);
413
414 ks = this->key->schedule;
415 t = _mm_xor_si128(cb, ks[0]);
416 for (round = 1; round < this->key->rounds; round++)
417 {
418 t = _mm_aesenc_si128(t, ks[round]);
419 }
420 t = _mm_aesenclast_si128(t, ks[this->key->rounds]);
421 b = _mm_xor_si128(t, b);
422
423 memcpy(out, &b, rem);
424
425 return y;
426 }
427
428 /**
429 * AES-128 GCM encryption/ICV generation
430 */
431 static void encrypt_gcm128(private_aesni_gcm_t *this,
432 size_t len, u_char *in, u_char *out, u_char *iv,
433 size_t alen, u_char *assoc, u_char *icv)
434 {
435 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
436 __m128i *ks, y, j, cb, *bi, *bo;
437 u_int blocks, pblocks, rem, i;
438
439 j = create_j(this, iv);
440 cb = increment_be(j);
441 y = icv_header(this, assoc, alen);
442 blocks = len / AES_BLOCK_SIZE;
443 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
444 rem = len % AES_BLOCK_SIZE;
445 bi = (__m128i*)in;
446 bo = (__m128i*)out;
447
448 ks = this->key->schedule;
449
450 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
451 {
452 d1 = _mm_loadu_si128(bi + i + 0);
453 d2 = _mm_loadu_si128(bi + i + 1);
454 d3 = _mm_loadu_si128(bi + i + 2);
455 d4 = _mm_loadu_si128(bi + i + 3);
456
457 t1 = _mm_xor_si128(cb, ks[0]);
458 cb = increment_be(cb);
459 t2 = _mm_xor_si128(cb, ks[0]);
460 cb = increment_be(cb);
461 t3 = _mm_xor_si128(cb, ks[0]);
462 cb = increment_be(cb);
463 t4 = _mm_xor_si128(cb, ks[0]);
464 cb = increment_be(cb);
465
466 t1 = _mm_aesenc_si128(t1, ks[1]);
467 t2 = _mm_aesenc_si128(t2, ks[1]);
468 t3 = _mm_aesenc_si128(t3, ks[1]);
469 t4 = _mm_aesenc_si128(t4, ks[1]);
470 t1 = _mm_aesenc_si128(t1, ks[2]);
471 t2 = _mm_aesenc_si128(t2, ks[2]);
472 t3 = _mm_aesenc_si128(t3, ks[2]);
473 t4 = _mm_aesenc_si128(t4, ks[2]);
474 t1 = _mm_aesenc_si128(t1, ks[3]);
475 t2 = _mm_aesenc_si128(t2, ks[3]);
476 t3 = _mm_aesenc_si128(t3, ks[3]);
477 t4 = _mm_aesenc_si128(t4, ks[3]);
478 t1 = _mm_aesenc_si128(t1, ks[4]);
479 t2 = _mm_aesenc_si128(t2, ks[4]);
480 t3 = _mm_aesenc_si128(t3, ks[4]);
481 t4 = _mm_aesenc_si128(t4, ks[4]);
482 t1 = _mm_aesenc_si128(t1, ks[5]);
483 t2 = _mm_aesenc_si128(t2, ks[5]);
484 t3 = _mm_aesenc_si128(t3, ks[5]);
485 t4 = _mm_aesenc_si128(t4, ks[5]);
486 t1 = _mm_aesenc_si128(t1, ks[6]);
487 t2 = _mm_aesenc_si128(t2, ks[6]);
488 t3 = _mm_aesenc_si128(t3, ks[6]);
489 t4 = _mm_aesenc_si128(t4, ks[6]);
490 t1 = _mm_aesenc_si128(t1, ks[7]);
491 t2 = _mm_aesenc_si128(t2, ks[7]);
492 t3 = _mm_aesenc_si128(t3, ks[7]);
493 t4 = _mm_aesenc_si128(t4, ks[7]);
494 t1 = _mm_aesenc_si128(t1, ks[8]);
495 t2 = _mm_aesenc_si128(t2, ks[8]);
496 t3 = _mm_aesenc_si128(t3, ks[8]);
497 t4 = _mm_aesenc_si128(t4, ks[8]);
498 t1 = _mm_aesenc_si128(t1, ks[9]);
499 t2 = _mm_aesenc_si128(t2, ks[9]);
500 t3 = _mm_aesenc_si128(t3, ks[9]);
501 t4 = _mm_aesenc_si128(t4, ks[9]);
502
503 t1 = _mm_aesenclast_si128(t1, ks[10]);
504 t2 = _mm_aesenclast_si128(t2, ks[10]);
505 t3 = _mm_aesenclast_si128(t3, ks[10]);
506 t4 = _mm_aesenclast_si128(t4, ks[10]);
507
508 t1 = _mm_xor_si128(t1, d1);
509 t2 = _mm_xor_si128(t2, d2);
510 t3 = _mm_xor_si128(t3, d3);
511 t4 = _mm_xor_si128(t4, d4);
512
513 y = _mm_xor_si128(y, t1);
514 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, t2, t3, t4);
515
516 _mm_storeu_si128(bo + i + 0, t1);
517 _mm_storeu_si128(bo + i + 1, t2);
518 _mm_storeu_si128(bo + i + 2, t3);
519 _mm_storeu_si128(bo + i + 3, t4);
520 }
521
522 for (i = pblocks; i < blocks; i++)
523 {
524 d1 = _mm_loadu_si128(bi + i);
525
526 t1 = _mm_xor_si128(cb, ks[0]);
527 t1 = _mm_aesenc_si128(t1, ks[1]);
528 t1 = _mm_aesenc_si128(t1, ks[2]);
529 t1 = _mm_aesenc_si128(t1, ks[3]);
530 t1 = _mm_aesenc_si128(t1, ks[4]);
531 t1 = _mm_aesenc_si128(t1, ks[5]);
532 t1 = _mm_aesenc_si128(t1, ks[6]);
533 t1 = _mm_aesenc_si128(t1, ks[7]);
534 t1 = _mm_aesenc_si128(t1, ks[8]);
535 t1 = _mm_aesenc_si128(t1, ks[9]);
536 t1 = _mm_aesenclast_si128(t1, ks[10]);
537
538 t1 = _mm_xor_si128(t1, d1);
539 _mm_storeu_si128(bo + i, t1);
540
541 y = ghash(this->h, y, t1);
542
543 cb = increment_be(cb);
544 }
545
546 if (rem)
547 {
548 y = encrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
549 }
550 y = icv_tailer(this, y, alen, len);
551 icv_crypt(this, y, j, icv);
552 }
553
554 /**
555 * AES-128 GCM decryption/ICV generation
556 */
557 static void decrypt_gcm128(private_aesni_gcm_t *this,
558 size_t len, u_char *in, u_char *out, u_char *iv,
559 size_t alen, u_char *assoc, u_char *icv)
560 {
561 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
562 __m128i *ks, y, j, cb, *bi, *bo;
563 u_int blocks, pblocks, rem, i;
564
565 j = create_j(this, iv);
566 cb = increment_be(j);
567 y = icv_header(this, assoc, alen);
568 blocks = len / AES_BLOCK_SIZE;
569 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
570 rem = len % AES_BLOCK_SIZE;
571 bi = (__m128i*)in;
572 bo = (__m128i*)out;
573
574 ks = this->key->schedule;
575
576 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
577 {
578 d1 = _mm_loadu_si128(bi + i + 0);
579 d2 = _mm_loadu_si128(bi + i + 1);
580 d3 = _mm_loadu_si128(bi + i + 2);
581 d4 = _mm_loadu_si128(bi + i + 3);
582
583 y = _mm_xor_si128(y, d1);
584 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, d2, d3, d4);
585
586 t1 = _mm_xor_si128(cb, ks[0]);
587 cb = increment_be(cb);
588 t2 = _mm_xor_si128(cb, ks[0]);
589 cb = increment_be(cb);
590 t3 = _mm_xor_si128(cb, ks[0]);
591 cb = increment_be(cb);
592 t4 = _mm_xor_si128(cb, ks[0]);
593 cb = increment_be(cb);
594
595 t1 = _mm_aesenc_si128(t1, ks[1]);
596 t2 = _mm_aesenc_si128(t2, ks[1]);
597 t3 = _mm_aesenc_si128(t3, ks[1]);
598 t4 = _mm_aesenc_si128(t4, ks[1]);
599 t1 = _mm_aesenc_si128(t1, ks[2]);
600 t2 = _mm_aesenc_si128(t2, ks[2]);
601 t3 = _mm_aesenc_si128(t3, ks[2]);
602 t4 = _mm_aesenc_si128(t4, ks[2]);
603 t1 = _mm_aesenc_si128(t1, ks[3]);
604 t2 = _mm_aesenc_si128(t2, ks[3]);
605 t3 = _mm_aesenc_si128(t3, ks[3]);
606 t4 = _mm_aesenc_si128(t4, ks[3]);
607 t1 = _mm_aesenc_si128(t1, ks[4]);
608 t2 = _mm_aesenc_si128(t2, ks[4]);
609 t3 = _mm_aesenc_si128(t3, ks[4]);
610 t4 = _mm_aesenc_si128(t4, ks[4]);
611 t1 = _mm_aesenc_si128(t1, ks[5]);
612 t2 = _mm_aesenc_si128(t2, ks[5]);
613 t3 = _mm_aesenc_si128(t3, ks[5]);
614 t4 = _mm_aesenc_si128(t4, ks[5]);
615 t1 = _mm_aesenc_si128(t1, ks[6]);
616 t2 = _mm_aesenc_si128(t2, ks[6]);
617 t3 = _mm_aesenc_si128(t3, ks[6]);
618 t4 = _mm_aesenc_si128(t4, ks[6]);
619 t1 = _mm_aesenc_si128(t1, ks[7]);
620 t2 = _mm_aesenc_si128(t2, ks[7]);
621 t3 = _mm_aesenc_si128(t3, ks[7]);
622 t4 = _mm_aesenc_si128(t4, ks[7]);
623 t1 = _mm_aesenc_si128(t1, ks[8]);
624 t2 = _mm_aesenc_si128(t2, ks[8]);
625 t3 = _mm_aesenc_si128(t3, ks[8]);
626 t4 = _mm_aesenc_si128(t4, ks[8]);
627 t1 = _mm_aesenc_si128(t1, ks[9]);
628 t2 = _mm_aesenc_si128(t2, ks[9]);
629 t3 = _mm_aesenc_si128(t3, ks[9]);
630 t4 = _mm_aesenc_si128(t4, ks[9]);
631
632 t1 = _mm_aesenclast_si128(t1, ks[10]);
633 t2 = _mm_aesenclast_si128(t2, ks[10]);
634 t3 = _mm_aesenclast_si128(t3, ks[10]);
635 t4 = _mm_aesenclast_si128(t4, ks[10]);
636
637 t1 = _mm_xor_si128(t1, d1);
638 t2 = _mm_xor_si128(t2, d2);
639 t3 = _mm_xor_si128(t3, d3);
640 t4 = _mm_xor_si128(t4, d4);
641
642 _mm_storeu_si128(bo + i + 0, t1);
643 _mm_storeu_si128(bo + i + 1, t2);
644 _mm_storeu_si128(bo + i + 2, t3);
645 _mm_storeu_si128(bo + i + 3, t4);
646 }
647
648 for (i = pblocks; i < blocks; i++)
649 {
650 d1 = _mm_loadu_si128(bi + i);
651
652 y = ghash(this->h, y, d1);
653
654 t1 = _mm_xor_si128(cb, ks[0]);
655 t1 = _mm_aesenc_si128(t1, ks[1]);
656 t1 = _mm_aesenc_si128(t1, ks[2]);
657 t1 = _mm_aesenc_si128(t1, ks[3]);
658 t1 = _mm_aesenc_si128(t1, ks[4]);
659 t1 = _mm_aesenc_si128(t1, ks[5]);
660 t1 = _mm_aesenc_si128(t1, ks[6]);
661 t1 = _mm_aesenc_si128(t1, ks[7]);
662 t1 = _mm_aesenc_si128(t1, ks[8]);
663 t1 = _mm_aesenc_si128(t1, ks[9]);
664 t1 = _mm_aesenclast_si128(t1, ks[10]);
665
666 t1 = _mm_xor_si128(t1, d1);
667 _mm_storeu_si128(bo + i, t1);
668
669 cb = increment_be(cb);
670 }
671
672 if (rem)
673 {
674 y = decrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
675 }
676 y = icv_tailer(this, y, alen, len);
677 icv_crypt(this, y, j, icv);
678 }
679
680 /**
681 * AES-192 GCM encryption/ICV generation
682 */
683 static void encrypt_gcm192(private_aesni_gcm_t *this,
684 size_t len, u_char *in, u_char *out, u_char *iv,
685 size_t alen, u_char *assoc, u_char *icv)
686 {
687 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
688 __m128i *ks, y, j, cb, *bi, *bo;
689 u_int blocks, pblocks, rem, i;
690
691 j = create_j(this, iv);
692 cb = increment_be(j);
693 y = icv_header(this, assoc, alen);
694 blocks = len / AES_BLOCK_SIZE;
695 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
696 rem = len % AES_BLOCK_SIZE;
697 bi = (__m128i*)in;
698 bo = (__m128i*)out;
699
700 ks = this->key->schedule;
701
702 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
703 {
704 d1 = _mm_loadu_si128(bi + i + 0);
705 d2 = _mm_loadu_si128(bi + i + 1);
706 d3 = _mm_loadu_si128(bi + i + 2);
707 d4 = _mm_loadu_si128(bi + i + 3);
708
709 t1 = _mm_xor_si128(cb, ks[0]);
710 cb = increment_be(cb);
711 t2 = _mm_xor_si128(cb, ks[0]);
712 cb = increment_be(cb);
713 t3 = _mm_xor_si128(cb, ks[0]);
714 cb = increment_be(cb);
715 t4 = _mm_xor_si128(cb, ks[0]);
716 cb = increment_be(cb);
717
718 t1 = _mm_aesenc_si128(t1, ks[1]);
719 t2 = _mm_aesenc_si128(t2, ks[1]);
720 t3 = _mm_aesenc_si128(t3, ks[1]);
721 t4 = _mm_aesenc_si128(t4, ks[1]);
722 t1 = _mm_aesenc_si128(t1, ks[2]);
723 t2 = _mm_aesenc_si128(t2, ks[2]);
724 t3 = _mm_aesenc_si128(t3, ks[2]);
725 t4 = _mm_aesenc_si128(t4, ks[2]);
726 t1 = _mm_aesenc_si128(t1, ks[3]);
727 t2 = _mm_aesenc_si128(t2, ks[3]);
728 t3 = _mm_aesenc_si128(t3, ks[3]);
729 t4 = _mm_aesenc_si128(t4, ks[3]);
730 t1 = _mm_aesenc_si128(t1, ks[4]);
731 t2 = _mm_aesenc_si128(t2, ks[4]);
732 t3 = _mm_aesenc_si128(t3, ks[4]);
733 t4 = _mm_aesenc_si128(t4, ks[4]);
734 t1 = _mm_aesenc_si128(t1, ks[5]);
735 t2 = _mm_aesenc_si128(t2, ks[5]);
736 t3 = _mm_aesenc_si128(t3, ks[5]);
737 t4 = _mm_aesenc_si128(t4, ks[5]);
738 t1 = _mm_aesenc_si128(t1, ks[6]);
739 t2 = _mm_aesenc_si128(t2, ks[6]);
740 t3 = _mm_aesenc_si128(t3, ks[6]);
741 t4 = _mm_aesenc_si128(t4, ks[6]);
742 t1 = _mm_aesenc_si128(t1, ks[7]);
743 t2 = _mm_aesenc_si128(t2, ks[7]);
744 t3 = _mm_aesenc_si128(t3, ks[7]);
745 t4 = _mm_aesenc_si128(t4, ks[7]);
746 t1 = _mm_aesenc_si128(t1, ks[8]);
747 t2 = _mm_aesenc_si128(t2, ks[8]);
748 t3 = _mm_aesenc_si128(t3, ks[8]);
749 t4 = _mm_aesenc_si128(t4, ks[8]);
750 t1 = _mm_aesenc_si128(t1, ks[9]);
751 t2 = _mm_aesenc_si128(t2, ks[9]);
752 t3 = _mm_aesenc_si128(t3, ks[9]);
753 t4 = _mm_aesenc_si128(t4, ks[9]);
754 t1 = _mm_aesenc_si128(t1, ks[10]);
755 t2 = _mm_aesenc_si128(t2, ks[10]);
756 t3 = _mm_aesenc_si128(t3, ks[10]);
757 t4 = _mm_aesenc_si128(t4, ks[10]);
758 t1 = _mm_aesenc_si128(t1, ks[11]);
759 t2 = _mm_aesenc_si128(t2, ks[11]);
760 t3 = _mm_aesenc_si128(t3, ks[11]);
761 t4 = _mm_aesenc_si128(t4, ks[11]);
762
763 t1 = _mm_aesenclast_si128(t1, ks[12]);
764 t2 = _mm_aesenclast_si128(t2, ks[12]);
765 t3 = _mm_aesenclast_si128(t3, ks[12]);
766 t4 = _mm_aesenclast_si128(t4, ks[12]);
767
768 t1 = _mm_xor_si128(t1, d1);
769 t2 = _mm_xor_si128(t2, d2);
770 t3 = _mm_xor_si128(t3, d3);
771 t4 = _mm_xor_si128(t4, d4);
772
773 y = _mm_xor_si128(y, t1);
774 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, t2, t3, t4);
775
776 _mm_storeu_si128(bo + i + 0, t1);
777 _mm_storeu_si128(bo + i + 1, t2);
778 _mm_storeu_si128(bo + i + 2, t3);
779 _mm_storeu_si128(bo + i + 3, t4);
780 }
781
782 for (i = pblocks; i < blocks; i++)
783 {
784 d1 = _mm_loadu_si128(bi + i);
785
786 t1 = _mm_xor_si128(cb, ks[0]);
787 t1 = _mm_aesenc_si128(t1, ks[1]);
788 t1 = _mm_aesenc_si128(t1, ks[2]);
789 t1 = _mm_aesenc_si128(t1, ks[3]);
790 t1 = _mm_aesenc_si128(t1, ks[4]);
791 t1 = _mm_aesenc_si128(t1, ks[5]);
792 t1 = _mm_aesenc_si128(t1, ks[6]);
793 t1 = _mm_aesenc_si128(t1, ks[7]);
794 t1 = _mm_aesenc_si128(t1, ks[8]);
795 t1 = _mm_aesenc_si128(t1, ks[9]);
796 t1 = _mm_aesenc_si128(t1, ks[10]);
797 t1 = _mm_aesenc_si128(t1, ks[11]);
798 t1 = _mm_aesenclast_si128(t1, ks[12]);
799
800 t1 = _mm_xor_si128(t1, d1);
801 _mm_storeu_si128(bo + i, t1);
802
803 y = ghash(this->h, y, t1);
804
805 cb = increment_be(cb);
806 }
807
808 if (rem)
809 {
810 y = encrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
811 }
812 y = icv_tailer(this, y, alen, len);
813 icv_crypt(this, y, j, icv);
814 }
815
816 /**
817 * AES-192 GCM decryption/ICV generation
818 */
819 static void decrypt_gcm192(private_aesni_gcm_t *this,
820 size_t len, u_char *in, u_char *out, u_char *iv,
821 size_t alen, u_char *assoc, u_char *icv)
822 {
823 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
824 __m128i *ks, y, j, cb, *bi, *bo;
825 u_int blocks, pblocks, rem, i;
826
827 j = create_j(this, iv);
828 cb = increment_be(j);
829 y = icv_header(this, assoc, alen);
830 blocks = len / AES_BLOCK_SIZE;
831 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
832 rem = len % AES_BLOCK_SIZE;
833 bi = (__m128i*)in;
834 bo = (__m128i*)out;
835
836 ks = this->key->schedule;
837
838 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
839 {
840 d1 = _mm_loadu_si128(bi + i + 0);
841 d2 = _mm_loadu_si128(bi + i + 1);
842 d3 = _mm_loadu_si128(bi + i + 2);
843 d4 = _mm_loadu_si128(bi + i + 3);
844
845 y = _mm_xor_si128(y, d1);
846 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, d2, d3, d4);
847
848 t1 = _mm_xor_si128(cb, ks[0]);
849 cb = increment_be(cb);
850 t2 = _mm_xor_si128(cb, ks[0]);
851 cb = increment_be(cb);
852 t3 = _mm_xor_si128(cb, ks[0]);
853 cb = increment_be(cb);
854 t4 = _mm_xor_si128(cb, ks[0]);
855 cb = increment_be(cb);
856
857 t1 = _mm_aesenc_si128(t1, ks[1]);
858 t2 = _mm_aesenc_si128(t2, ks[1]);
859 t3 = _mm_aesenc_si128(t3, ks[1]);
860 t4 = _mm_aesenc_si128(t4, ks[1]);
861 t1 = _mm_aesenc_si128(t1, ks[2]);
862 t2 = _mm_aesenc_si128(t2, ks[2]);
863 t3 = _mm_aesenc_si128(t3, ks[2]);
864 t4 = _mm_aesenc_si128(t4, ks[2]);
865 t1 = _mm_aesenc_si128(t1, ks[3]);
866 t2 = _mm_aesenc_si128(t2, ks[3]);
867 t3 = _mm_aesenc_si128(t3, ks[3]);
868 t4 = _mm_aesenc_si128(t4, ks[3]);
869 t1 = _mm_aesenc_si128(t1, ks[4]);
870 t2 = _mm_aesenc_si128(t2, ks[4]);
871 t3 = _mm_aesenc_si128(t3, ks[4]);
872 t4 = _mm_aesenc_si128(t4, ks[4]);
873 t1 = _mm_aesenc_si128(t1, ks[5]);
874 t2 = _mm_aesenc_si128(t2, ks[5]);
875 t3 = _mm_aesenc_si128(t3, ks[5]);
876 t4 = _mm_aesenc_si128(t4, ks[5]);
877 t1 = _mm_aesenc_si128(t1, ks[6]);
878 t2 = _mm_aesenc_si128(t2, ks[6]);
879 t3 = _mm_aesenc_si128(t3, ks[6]);
880 t4 = _mm_aesenc_si128(t4, ks[6]);
881 t1 = _mm_aesenc_si128(t1, ks[7]);
882 t2 = _mm_aesenc_si128(t2, ks[7]);
883 t3 = _mm_aesenc_si128(t3, ks[7]);
884 t4 = _mm_aesenc_si128(t4, ks[7]);
885 t1 = _mm_aesenc_si128(t1, ks[8]);
886 t2 = _mm_aesenc_si128(t2, ks[8]);
887 t3 = _mm_aesenc_si128(t3, ks[8]);
888 t4 = _mm_aesenc_si128(t4, ks[8]);
889 t1 = _mm_aesenc_si128(t1, ks[9]);
890 t2 = _mm_aesenc_si128(t2, ks[9]);
891 t3 = _mm_aesenc_si128(t3, ks[9]);
892 t4 = _mm_aesenc_si128(t4, ks[9]);
893 t1 = _mm_aesenc_si128(t1, ks[10]);
894 t2 = _mm_aesenc_si128(t2, ks[10]);
895 t3 = _mm_aesenc_si128(t3, ks[10]);
896 t4 = _mm_aesenc_si128(t4, ks[10]);
897 t1 = _mm_aesenc_si128(t1, ks[11]);
898 t2 = _mm_aesenc_si128(t2, ks[11]);
899 t3 = _mm_aesenc_si128(t3, ks[11]);
900 t4 = _mm_aesenc_si128(t4, ks[11]);
901
902 t1 = _mm_aesenclast_si128(t1, ks[12]);
903 t2 = _mm_aesenclast_si128(t2, ks[12]);
904 t3 = _mm_aesenclast_si128(t3, ks[12]);
905 t4 = _mm_aesenclast_si128(t4, ks[12]);
906
907 t1 = _mm_xor_si128(t1, d1);
908 t2 = _mm_xor_si128(t2, d2);
909 t3 = _mm_xor_si128(t3, d3);
910 t4 = _mm_xor_si128(t4, d4);
911
912 _mm_storeu_si128(bo + i + 0, t1);
913 _mm_storeu_si128(bo + i + 1, t2);
914 _mm_storeu_si128(bo + i + 2, t3);
915 _mm_storeu_si128(bo + i + 3, t4);
916 }
917
918 for (i = pblocks; i < blocks; i++)
919 {
920 d1 = _mm_loadu_si128(bi + i);
921
922 y = ghash(this->h, y, d1);
923
924 t1 = _mm_xor_si128(cb, ks[0]);
925 t1 = _mm_aesenc_si128(t1, ks[1]);
926 t1 = _mm_aesenc_si128(t1, ks[2]);
927 t1 = _mm_aesenc_si128(t1, ks[3]);
928 t1 = _mm_aesenc_si128(t1, ks[4]);
929 t1 = _mm_aesenc_si128(t1, ks[5]);
930 t1 = _mm_aesenc_si128(t1, ks[6]);
931 t1 = _mm_aesenc_si128(t1, ks[7]);
932 t1 = _mm_aesenc_si128(t1, ks[8]);
933 t1 = _mm_aesenc_si128(t1, ks[9]);
934 t1 = _mm_aesenc_si128(t1, ks[10]);
935 t1 = _mm_aesenc_si128(t1, ks[11]);
936 t1 = _mm_aesenclast_si128(t1, ks[12]);
937
938 t1 = _mm_xor_si128(t1, d1);
939 _mm_storeu_si128(bo + i, t1);
940
941 cb = increment_be(cb);
942 }
943
944 if (rem)
945 {
946 y = decrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
947 }
948 y = icv_tailer(this, y, alen, len);
949 icv_crypt(this, y, j, icv);
950 }
951
952 /**
953 * AES-256 GCM encryption/ICV generation
954 */
955 static void encrypt_gcm256(private_aesni_gcm_t *this,
956 size_t len, u_char *in, u_char *out, u_char *iv,
957 size_t alen, u_char *assoc, u_char *icv)
958 {
959 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
960 __m128i *ks, y, j, cb, *bi, *bo;
961 u_int blocks, pblocks, rem, i;
962
963 j = create_j(this, iv);
964 cb = increment_be(j);
965 y = icv_header(this, assoc, alen);
966 blocks = len / AES_BLOCK_SIZE;
967 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
968 rem = len % AES_BLOCK_SIZE;
969 bi = (__m128i*)in;
970 bo = (__m128i*)out;
971
972 ks = this->key->schedule;
973
974 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
975 {
976 d1 = _mm_loadu_si128(bi + i + 0);
977 d2 = _mm_loadu_si128(bi + i + 1);
978 d3 = _mm_loadu_si128(bi + i + 2);
979 d4 = _mm_loadu_si128(bi + i + 3);
980
981 t1 = _mm_xor_si128(cb, ks[0]);
982 cb = increment_be(cb);
983 t2 = _mm_xor_si128(cb, ks[0]);
984 cb = increment_be(cb);
985 t3 = _mm_xor_si128(cb, ks[0]);
986 cb = increment_be(cb);
987 t4 = _mm_xor_si128(cb, ks[0]);
988 cb = increment_be(cb);
989
990 t1 = _mm_aesenc_si128(t1, ks[1]);
991 t2 = _mm_aesenc_si128(t2, ks[1]);
992 t3 = _mm_aesenc_si128(t3, ks[1]);
993 t4 = _mm_aesenc_si128(t4, ks[1]);
994 t1 = _mm_aesenc_si128(t1, ks[2]);
995 t2 = _mm_aesenc_si128(t2, ks[2]);
996 t3 = _mm_aesenc_si128(t3, ks[2]);
997 t4 = _mm_aesenc_si128(t4, ks[2]);
998 t1 = _mm_aesenc_si128(t1, ks[3]);
999 t2 = _mm_aesenc_si128(t2, ks[3]);
1000 t3 = _mm_aesenc_si128(t3, ks[3]);
1001 t4 = _mm_aesenc_si128(t4, ks[3]);
1002 t1 = _mm_aesenc_si128(t1, ks[4]);
1003 t2 = _mm_aesenc_si128(t2, ks[4]);
1004 t3 = _mm_aesenc_si128(t3, ks[4]);
1005 t4 = _mm_aesenc_si128(t4, ks[4]);
1006 t1 = _mm_aesenc_si128(t1, ks[5]);
1007 t2 = _mm_aesenc_si128(t2, ks[5]);
1008 t3 = _mm_aesenc_si128(t3, ks[5]);
1009 t4 = _mm_aesenc_si128(t4, ks[5]);
1010 t1 = _mm_aesenc_si128(t1, ks[6]);
1011 t2 = _mm_aesenc_si128(t2, ks[6]);
1012 t3 = _mm_aesenc_si128(t3, ks[6]);
1013 t4 = _mm_aesenc_si128(t4, ks[6]);
1014 t1 = _mm_aesenc_si128(t1, ks[7]);
1015 t2 = _mm_aesenc_si128(t2, ks[7]);
1016 t3 = _mm_aesenc_si128(t3, ks[7]);
1017 t4 = _mm_aesenc_si128(t4, ks[7]);
1018 t1 = _mm_aesenc_si128(t1, ks[8]);
1019 t2 = _mm_aesenc_si128(t2, ks[8]);
1020 t3 = _mm_aesenc_si128(t3, ks[8]);
1021 t4 = _mm_aesenc_si128(t4, ks[8]);
1022 t1 = _mm_aesenc_si128(t1, ks[9]);
1023 t2 = _mm_aesenc_si128(t2, ks[9]);
1024 t3 = _mm_aesenc_si128(t3, ks[9]);
1025 t4 = _mm_aesenc_si128(t4, ks[9]);
1026 t1 = _mm_aesenc_si128(t1, ks[10]);
1027 t2 = _mm_aesenc_si128(t2, ks[10]);
1028 t3 = _mm_aesenc_si128(t3, ks[10]);
1029 t4 = _mm_aesenc_si128(t4, ks[10]);
1030 t1 = _mm_aesenc_si128(t1, ks[11]);
1031 t2 = _mm_aesenc_si128(t2, ks[11]);
1032 t3 = _mm_aesenc_si128(t3, ks[11]);
1033 t4 = _mm_aesenc_si128(t4, ks[11]);
1034 t1 = _mm_aesenc_si128(t1, ks[12]);
1035 t2 = _mm_aesenc_si128(t2, ks[12]);
1036 t3 = _mm_aesenc_si128(t3, ks[12]);
1037 t4 = _mm_aesenc_si128(t4, ks[12]);
1038 t1 = _mm_aesenc_si128(t1, ks[13]);
1039 t2 = _mm_aesenc_si128(t2, ks[13]);
1040 t3 = _mm_aesenc_si128(t3, ks[13]);
1041 t4 = _mm_aesenc_si128(t4, ks[13]);
1042
1043 t1 = _mm_aesenclast_si128(t1, ks[14]);
1044 t2 = _mm_aesenclast_si128(t2, ks[14]);
1045 t3 = _mm_aesenclast_si128(t3, ks[14]);
1046 t4 = _mm_aesenclast_si128(t4, ks[14]);
1047
1048 t1 = _mm_xor_si128(t1, d1);
1049 t2 = _mm_xor_si128(t2, d2);
1050 t3 = _mm_xor_si128(t3, d3);
1051 t4 = _mm_xor_si128(t4, d4);
1052
1053 y = _mm_xor_si128(y, t1);
1054 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, t2, t3, t4);
1055
1056 _mm_storeu_si128(bo + i + 0, t1);
1057 _mm_storeu_si128(bo + i + 1, t2);
1058 _mm_storeu_si128(bo + i + 2, t3);
1059 _mm_storeu_si128(bo + i + 3, t4);
1060 }
1061
1062 for (i = pblocks; i < blocks; i++)
1063 {
1064 d1 = _mm_loadu_si128(bi + i);
1065
1066 t1 = _mm_xor_si128(cb, ks[0]);
1067 t1 = _mm_aesenc_si128(t1, ks[1]);
1068 t1 = _mm_aesenc_si128(t1, ks[2]);
1069 t1 = _mm_aesenc_si128(t1, ks[3]);
1070 t1 = _mm_aesenc_si128(t1, ks[4]);
1071 t1 = _mm_aesenc_si128(t1, ks[5]);
1072 t1 = _mm_aesenc_si128(t1, ks[6]);
1073 t1 = _mm_aesenc_si128(t1, ks[7]);
1074 t1 = _mm_aesenc_si128(t1, ks[8]);
1075 t1 = _mm_aesenc_si128(t1, ks[9]);
1076 t1 = _mm_aesenc_si128(t1, ks[10]);
1077 t1 = _mm_aesenc_si128(t1, ks[11]);
1078 t1 = _mm_aesenc_si128(t1, ks[12]);
1079 t1 = _mm_aesenc_si128(t1, ks[13]);
1080 t1 = _mm_aesenclast_si128(t1, ks[14]);
1081
1082 t1 = _mm_xor_si128(t1, d1);
1083 _mm_storeu_si128(bo + i, t1);
1084
1085 y = ghash(this->h, y, t1);
1086
1087 cb = increment_be(cb);
1088 }
1089
1090 if (rem)
1091 {
1092 y = encrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
1093 }
1094 y = icv_tailer(this, y, alen, len);
1095 icv_crypt(this, y, j, icv);
1096 }
1097
1098 /**
1099 * AES-256 GCM decryption/ICV generation
1100 */
1101 static void decrypt_gcm256(private_aesni_gcm_t *this,
1102 size_t len, u_char *in, u_char *out, u_char *iv,
1103 size_t alen, u_char *assoc, u_char *icv)
1104 {
1105 __m128i d1, d2, d3, d4, t1, t2, t3, t4;
1106 __m128i *ks, y, j, cb, *bi, *bo;
1107 u_int blocks, pblocks, rem, i;
1108
1109 j = create_j(this, iv);
1110 cb = increment_be(j);
1111 y = icv_header(this, assoc, alen);
1112 blocks = len / AES_BLOCK_SIZE;
1113 pblocks = blocks - (blocks % GCM_CRYPT_PARALLELISM);
1114 rem = len % AES_BLOCK_SIZE;
1115 bi = (__m128i*)in;
1116 bo = (__m128i*)out;
1117
1118 ks = this->key->schedule;
1119
1120 for (i = 0; i < pblocks; i += GCM_CRYPT_PARALLELISM)
1121 {
1122 d1 = _mm_loadu_si128(bi + i + 0);
1123 d2 = _mm_loadu_si128(bi + i + 1);
1124 d3 = _mm_loadu_si128(bi + i + 2);
1125 d4 = _mm_loadu_si128(bi + i + 3);
1126
1127 y = _mm_xor_si128(y, d1);
1128 y = mult4xor(this->hhhh, this->hhh, this->hh, this->h, y, d2, d3, d4);
1129
1130 t1 = _mm_xor_si128(cb, ks[0]);
1131 cb = increment_be(cb);
1132 t2 = _mm_xor_si128(cb, ks[0]);
1133 cb = increment_be(cb);
1134 t3 = _mm_xor_si128(cb, ks[0]);
1135 cb = increment_be(cb);
1136 t4 = _mm_xor_si128(cb, ks[0]);
1137 cb = increment_be(cb);
1138
1139 t1 = _mm_aesenc_si128(t1, ks[1]);
1140 t2 = _mm_aesenc_si128(t2, ks[1]);
1141 t3 = _mm_aesenc_si128(t3, ks[1]);
1142 t4 = _mm_aesenc_si128(t4, ks[1]);
1143 t1 = _mm_aesenc_si128(t1, ks[2]);
1144 t2 = _mm_aesenc_si128(t2, ks[2]);
1145 t3 = _mm_aesenc_si128(t3, ks[2]);
1146 t4 = _mm_aesenc_si128(t4, ks[2]);
1147 t1 = _mm_aesenc_si128(t1, ks[3]);
1148 t2 = _mm_aesenc_si128(t2, ks[3]);
1149 t3 = _mm_aesenc_si128(t3, ks[3]);
1150 t4 = _mm_aesenc_si128(t4, ks[3]);
1151 t1 = _mm_aesenc_si128(t1, ks[4]);
1152 t2 = _mm_aesenc_si128(t2, ks[4]);
1153 t3 = _mm_aesenc_si128(t3, ks[4]);
1154 t4 = _mm_aesenc_si128(t4, ks[4]);
1155 t1 = _mm_aesenc_si128(t1, ks[5]);
1156 t2 = _mm_aesenc_si128(t2, ks[5]);
1157 t3 = _mm_aesenc_si128(t3, ks[5]);
1158 t4 = _mm_aesenc_si128(t4, ks[5]);
1159 t1 = _mm_aesenc_si128(t1, ks[6]);
1160 t2 = _mm_aesenc_si128(t2, ks[6]);
1161 t3 = _mm_aesenc_si128(t3, ks[6]);
1162 t4 = _mm_aesenc_si128(t4, ks[6]);
1163 t1 = _mm_aesenc_si128(t1, ks[7]);
1164 t2 = _mm_aesenc_si128(t2, ks[7]);
1165 t3 = _mm_aesenc_si128(t3, ks[7]);
1166 t4 = _mm_aesenc_si128(t4, ks[7]);
1167 t1 = _mm_aesenc_si128(t1, ks[8]);
1168 t2 = _mm_aesenc_si128(t2, ks[8]);
1169 t3 = _mm_aesenc_si128(t3, ks[8]);
1170 t4 = _mm_aesenc_si128(t4, ks[8]);
1171 t1 = _mm_aesenc_si128(t1, ks[9]);
1172 t2 = _mm_aesenc_si128(t2, ks[9]);
1173 t3 = _mm_aesenc_si128(t3, ks[9]);
1174 t4 = _mm_aesenc_si128(t4, ks[9]);
1175 t1 = _mm_aesenc_si128(t1, ks[10]);
1176 t2 = _mm_aesenc_si128(t2, ks[10]);
1177 t3 = _mm_aesenc_si128(t3, ks[10]);
1178 t4 = _mm_aesenc_si128(t4, ks[10]);
1179 t1 = _mm_aesenc_si128(t1, ks[11]);
1180 t2 = _mm_aesenc_si128(t2, ks[11]);
1181 t3 = _mm_aesenc_si128(t3, ks[11]);
1182 t4 = _mm_aesenc_si128(t4, ks[11]);
1183 t1 = _mm_aesenc_si128(t1, ks[12]);
1184 t2 = _mm_aesenc_si128(t2, ks[12]);
1185 t3 = _mm_aesenc_si128(t3, ks[12]);
1186 t4 = _mm_aesenc_si128(t4, ks[12]);
1187 t1 = _mm_aesenc_si128(t1, ks[13]);
1188 t2 = _mm_aesenc_si128(t2, ks[13]);
1189 t3 = _mm_aesenc_si128(t3, ks[13]);
1190 t4 = _mm_aesenc_si128(t4, ks[13]);
1191
1192 t1 = _mm_aesenclast_si128(t1, ks[14]);
1193 t2 = _mm_aesenclast_si128(t2, ks[14]);
1194 t3 = _mm_aesenclast_si128(t3, ks[14]);
1195 t4 = _mm_aesenclast_si128(t4, ks[14]);
1196
1197 t1 = _mm_xor_si128(t1, d1);
1198 t2 = _mm_xor_si128(t2, d2);
1199 t3 = _mm_xor_si128(t3, d3);
1200 t4 = _mm_xor_si128(t4, d4);
1201
1202 _mm_storeu_si128(bo + i + 0, t1);
1203 _mm_storeu_si128(bo + i + 1, t2);
1204 _mm_storeu_si128(bo + i + 2, t3);
1205 _mm_storeu_si128(bo + i + 3, t4);
1206 }
1207
1208 for (i = pblocks; i < blocks; i++)
1209 {
1210 d1 = _mm_loadu_si128(bi + i);
1211
1212 y = ghash(this->h, y, d1);
1213
1214 t1 = _mm_xor_si128(cb, ks[0]);
1215 t1 = _mm_aesenc_si128(t1, ks[1]);
1216 t1 = _mm_aesenc_si128(t1, ks[2]);
1217 t1 = _mm_aesenc_si128(t1, ks[3]);
1218 t1 = _mm_aesenc_si128(t1, ks[4]);
1219 t1 = _mm_aesenc_si128(t1, ks[5]);
1220 t1 = _mm_aesenc_si128(t1, ks[6]);
1221 t1 = _mm_aesenc_si128(t1, ks[7]);
1222 t1 = _mm_aesenc_si128(t1, ks[8]);
1223 t1 = _mm_aesenc_si128(t1, ks[9]);
1224 t1 = _mm_aesenc_si128(t1, ks[10]);
1225 t1 = _mm_aesenc_si128(t1, ks[11]);
1226 t1 = _mm_aesenc_si128(t1, ks[12]);
1227 t1 = _mm_aesenc_si128(t1, ks[13]);
1228 t1 = _mm_aesenclast_si128(t1, ks[14]);
1229
1230 t1 = _mm_xor_si128(t1, d1);
1231 _mm_storeu_si128(bo + i, t1);
1232
1233 cb = increment_be(cb);
1234 }
1235
1236 if (rem)
1237 {
1238 y = decrypt_gcm_rem(this, rem, bi + blocks, bo + blocks, cb, y);
1239 }
1240 y = icv_tailer(this, y, alen, len);
1241 icv_crypt(this, y, j, icv);
1242 }
1243
1244 METHOD(aead_t, encrypt, bool,
1245 private_aesni_gcm_t *this, chunk_t plain, chunk_t assoc, chunk_t iv,
1246 chunk_t *encr)
1247 {
1248 u_char *out;
1249
1250 if (!this->key || iv.len != IV_SIZE)
1251 {
1252 return FALSE;
1253 }
1254 out = plain.ptr;
1255 if (encr)
1256 {
1257 *encr = chunk_alloc(plain.len + this->icv_size);
1258 out = encr->ptr;
1259 }
1260 this->encrypt(this, plain.len, plain.ptr, out, iv.ptr,
1261 assoc.len, assoc.ptr, out + plain.len);
1262 return TRUE;
1263 }
1264
1265 METHOD(aead_t, decrypt, bool,
1266 private_aesni_gcm_t *this, chunk_t encr, chunk_t assoc, chunk_t iv,
1267 chunk_t *plain)
1268 {
1269 u_char *out, icv[this->icv_size];
1270
1271 if (!this->key || iv.len != IV_SIZE || encr.len < this->icv_size)
1272 {
1273 return FALSE;
1274 }
1275 encr.len -= this->icv_size;
1276 out = encr.ptr;
1277 if (plain)
1278 {
1279 *plain = chunk_alloc(encr.len);
1280 out = plain->ptr;
1281 }
1282 this->decrypt(this, encr.len, encr.ptr, out, iv.ptr,
1283 assoc.len, assoc.ptr, icv);
1284 return memeq_const(icv, encr.ptr + encr.len, this->icv_size);
1285 }
1286
1287 METHOD(aead_t, get_block_size, size_t,
1288 private_aesni_gcm_t *this)
1289 {
1290 return 1;
1291 }
1292
1293 METHOD(aead_t, get_icv_size, size_t,
1294 private_aesni_gcm_t *this)
1295 {
1296 return this->icv_size;
1297 }
1298
1299 METHOD(aead_t, get_iv_size, size_t,
1300 private_aesni_gcm_t *this)
1301 {
1302 return IV_SIZE;
1303 }
1304
1305 METHOD(aead_t, get_iv_gen, iv_gen_t*,
1306 private_aesni_gcm_t *this)
1307 {
1308 return this->iv_gen;
1309 }
1310
1311 METHOD(aead_t, get_key_size, size_t,
1312 private_aesni_gcm_t *this)
1313 {
1314 return this->key_size + SALT_SIZE;
1315 }
1316
1317 METHOD(aead_t, set_key, bool,
1318 private_aesni_gcm_t *this, chunk_t key)
1319 {
1320 u_int round;
1321 __m128i *ks, h;
1322
1323 if (key.len != this->key_size + SALT_SIZE)
1324 {
1325 return FALSE;
1326 }
1327
1328 memcpy(this->salt, key.ptr + key.len - SALT_SIZE, SALT_SIZE);
1329 key.len -= SALT_SIZE;
1330
1331 DESTROY_IF(this->key);
1332 this->key = aesni_key_create(TRUE, key);
1333
1334 ks = this->key->schedule;
1335 h = _mm_xor_si128(_mm_setzero_si128(), ks[0]);
1336 for (round = 1; round < this->key->rounds; round++)
1337 {
1338 h = _mm_aesenc_si128(h, ks[round]);
1339 }
1340 h = _mm_aesenclast_si128(h, ks[this->key->rounds]);
1341
1342 this->h = h;
1343 h = swap128(h);
1344 this->hh = mult_block(h, this->h);
1345 this->hhh = mult_block(h, this->hh);
1346 this->hhhh = mult_block(h, this->hhh);
1347 this->h = swap128(this->h);
1348 this->hh = swap128(this->hh);
1349 this->hhh = swap128(this->hhh);
1350 this->hhhh = swap128(this->hhhh);
1351
1352 return TRUE;
1353 }
1354
1355 METHOD(aead_t, destroy, void,
1356 private_aesni_gcm_t *this)
1357 {
1358 DESTROY_IF(this->key);
1359 memwipe(&this->h, sizeof(this->h));
1360 memwipe(&this->hh, sizeof(this->hh));
1361 memwipe(&this->hhh, sizeof(this->hhh));
1362 memwipe(&this->hhhh, sizeof(this->hhhh));
1363 this->iv_gen->destroy(this->iv_gen);
1364 free_align(this);
1365 }
1366
1367 /**
1368 * See header
1369 */
1370 aesni_gcm_t *aesni_gcm_create(encryption_algorithm_t algo,
1371 size_t key_size, size_t salt_size)
1372 {
1373 private_aesni_gcm_t *this;
1374 size_t icv_size;
1375
1376 switch (key_size)
1377 {
1378 case 0:
1379 key_size = 16;
1380 break;
1381 case 16:
1382 case 24:
1383 case 32:
1384 break;
1385 default:
1386 return NULL;
1387 }
1388 if (salt_size && salt_size != SALT_SIZE)
1389 {
1390 /* currently not supported */
1391 return NULL;
1392 }
1393 switch (algo)
1394 {
1395 case ENCR_AES_GCM_ICV8:
1396 algo = ENCR_AES_CBC;
1397 icv_size = 8;
1398 break;
1399 case ENCR_AES_GCM_ICV12:
1400 algo = ENCR_AES_CBC;
1401 icv_size = 12;
1402 break;
1403 case ENCR_AES_GCM_ICV16:
1404 algo = ENCR_AES_CBC;
1405 icv_size = 16;
1406 break;
1407 default:
1408 return NULL;
1409 }
1410
1411 INIT_ALIGN(this, sizeof(__m128i),
1412 .public = {
1413 .aead = {
1414 .encrypt = _encrypt,
1415 .decrypt = _decrypt,
1416 .get_block_size = _get_block_size,
1417 .get_icv_size = _get_icv_size,
1418 .get_iv_size = _get_iv_size,
1419 .get_iv_gen = _get_iv_gen,
1420 .get_key_size = _get_key_size,
1421 .set_key = _set_key,
1422 .destroy = _destroy,
1423 },
1424 },
1425 .key_size = key_size,
1426 .iv_gen = iv_gen_seq_create(),
1427 .icv_size = icv_size,
1428 );
1429
1430 switch (key_size)
1431 {
1432 case 16:
1433 this->encrypt = encrypt_gcm128;
1434 this->decrypt = decrypt_gcm128;
1435 break;
1436 case 24:
1437 this->encrypt = encrypt_gcm192;
1438 this->decrypt = decrypt_gcm192;
1439 break;
1440 case 32:
1441 this->encrypt = encrypt_gcm256;
1442 this->decrypt = decrypt_gcm256;
1443 break;
1444 }
1445
1446 return &this->public;
1447 }