Line data Source code
1 : /*
2 : * Copyright (C) 2011-2016 Free Software Foundation, Inc.
3 : * Copyright (C) 2015-2018 Red Hat, Inc.
4 : *
5 : * Author: Nikos Mavrogiannopoulos
6 : *
7 : * This file is part of GnuTLS.
8 : *
9 : * The GnuTLS is free software; you can redistribute it and/or
10 : * modify it under the terms of the GNU Lesser General Public License
11 : * as published by the Free Software Foundation; either version 2.1 of
12 : * the License, or (at your option) any later version.
13 : *
14 : * This library is distributed in the hope that it will be useful, but
15 : * WITHOUT ANY WARRANTY; without even the implied warranty of
16 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 : * Lesser General Public License for more details.
18 : *
19 : * You should have received a copy of the GNU Lesser General Public License
20 : * along with this program. If not, see <https://www.gnu.org/licenses/>
21 : *
22 : */
23 :
24 : /*
25 : * The following code is an implementation of the AES-128-GCM cipher
26 : * using intel's AES instruction set.
27 : */
28 :
29 : #include "errors.h"
30 : #include "gnutls_int.h"
31 : #include <gnutls/crypto.h>
32 : #include "errors.h"
33 : #include <aes-x86.h>
34 : #include <x86-common.h>
35 : #include <nettle/memxor.h>
36 : #include <byteswap.h>
37 :
38 : #define GCM_BLOCK_SIZE 16
39 :
40 : /* GCM mode with PCLMUL and AVX optimization */
41 :
42 : typedef struct {
43 : uint64_t hi, lo;
44 : } u128;
45 :
46 : /* This is the gcm128 structure used in openssl. It
47 : * is compatible with the included assembly code.
48 : */
49 : struct gcm128_context {
50 : union {
51 : uint64_t u[2];
52 : uint32_t d[4];
53 : uint8_t c[16];
54 : size_t t[16 / sizeof(size_t)];
55 : } Yi, EKi, EK0, len, Xi, H;
56 : u128 Htable[16];
57 : };
58 :
59 : struct aes_gcm_ctx {
60 : AES_KEY expanded_key;
61 : struct gcm128_context gcm;
62 : unsigned finished;
63 : unsigned auth_finished;
64 : };
65 :
66 : void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
67 : void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, size_t len);
68 : void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]);
69 :
70 27916 : static void aes_gcm_deinit(void *_ctx)
71 : {
72 27916 : struct aes_gcm_ctx *ctx = _ctx;
73 :
74 27916 : zeroize_temp_key(ctx, sizeof(*ctx));
75 27916 : gnutls_free(ctx);
76 27916 : }
77 :
78 : static int
79 27968 : aes_gcm_cipher_init(gnutls_cipher_algorithm_t algorithm, void **_ctx,
80 : int enc)
81 : {
82 : /* we use key size to distinguish */
83 27968 : if (algorithm != GNUTLS_CIPHER_AES_128_GCM &&
84 27968 : algorithm != GNUTLS_CIPHER_AES_192_GCM &&
85 : algorithm != GNUTLS_CIPHER_AES_256_GCM)
86 : return GNUTLS_E_INVALID_REQUEST;
87 :
88 27968 : *_ctx = gnutls_calloc(1, sizeof(struct aes_gcm_ctx));
89 27968 : if (*_ctx == NULL) {
90 0 : gnutls_assert();
91 0 : return GNUTLS_E_MEMORY_ERROR;
92 : }
93 :
94 : return 0;
95 : }
96 :
97 : static int
98 27968 : aes_gcm_cipher_setkey(void *_ctx, const void *userkey, size_t keysize)
99 : {
100 27968 : struct aes_gcm_ctx *ctx = _ctx;
101 27968 : int ret;
102 :
103 27968 : CHECK_AES_KEYSIZE(keysize);
104 :
105 27968 : ret =
106 55936 : aesni_set_encrypt_key(userkey, keysize * 8,
107 27968 : ALIGN16(&ctx->expanded_key));
108 27968 : if (ret != 0)
109 0 : return gnutls_assert_val(GNUTLS_E_ENCRYPTION_FAILED);
110 :
111 27968 : aesni_ecb_encrypt(ctx->gcm.H.c, ctx->gcm.H.c,
112 : GCM_BLOCK_SIZE, ALIGN16(&ctx->expanded_key), 1);
113 :
114 27968 : ctx->gcm.H.u[0] = bswap_64(ctx->gcm.H.u[0]);
115 27968 : ctx->gcm.H.u[1] = bswap_64(ctx->gcm.H.u[1]);
116 :
117 27968 : gcm_init_avx(ctx->gcm.Htable, ctx->gcm.H.u);
118 :
119 27968 : return 0;
120 : }
121 :
122 1417690 : static int aes_gcm_setiv(void *_ctx, const void *iv, size_t iv_size)
123 : {
124 1417690 : struct aes_gcm_ctx *ctx = _ctx;
125 :
126 1417690 : if (iv_size != GCM_BLOCK_SIZE - 4)
127 0 : return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
128 :
129 1417690 : memset(ctx->gcm.Xi.c, 0, sizeof(ctx->gcm.Xi.c));
130 1417690 : memset(ctx->gcm.len.c, 0, sizeof(ctx->gcm.len.c));
131 :
132 1417690 : memcpy(ctx->gcm.Yi.c, iv, GCM_BLOCK_SIZE - 4);
133 1417690 : ctx->gcm.Yi.c[GCM_BLOCK_SIZE - 4] = 0;
134 1417690 : ctx->gcm.Yi.c[GCM_BLOCK_SIZE - 3] = 0;
135 1417690 : ctx->gcm.Yi.c[GCM_BLOCK_SIZE - 2] = 0;
136 1417690 : ctx->gcm.Yi.c[GCM_BLOCK_SIZE - 1] = 1;
137 :
138 1417690 : aesni_ecb_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c,
139 1417690 : GCM_BLOCK_SIZE, ALIGN16(&ctx->expanded_key), 1);
140 1417690 : ctx->gcm.Yi.c[GCM_BLOCK_SIZE - 1] = 2;
141 1417690 : ctx->finished = 0;
142 1417690 : ctx->auth_finished = 0;
143 1417690 : return 0;
144 : }
145 :
146 : static void
147 2989950 : gcm_ghash(struct aes_gcm_ctx *ctx, const uint8_t * src, size_t src_size)
148 : {
149 2989950 : size_t rest = src_size % GCM_BLOCK_SIZE;
150 2989950 : size_t aligned_size = src_size - rest;
151 :
152 2989950 : if (aligned_size > 0)
153 769278 : gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, src,
154 : aligned_size);
155 :
156 2989950 : if (rest > 0) {
157 2775960 : memxor(ctx->gcm.Xi.c, src + aligned_size, rest);
158 2775960 : gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
159 : }
160 2989950 : }
161 :
162 : static inline void
163 1358360 : ctr_encrypt_last(struct aes_gcm_ctx *ctx, const uint8_t * src,
164 : uint8_t * dst, size_t pos, size_t length)
165 : {
166 1358360 : uint8_t tmp[GCM_BLOCK_SIZE];
167 1358360 : uint8_t out[GCM_BLOCK_SIZE];
168 :
169 1358360 : memcpy(tmp, &src[pos], length);
170 1358360 : aesni_ctr32_encrypt_blocks(tmp, out, 1,
171 1358360 : ALIGN16(&ctx->expanded_key),
172 1358360 : ctx->gcm.Yi.c);
173 :
174 1358360 : memcpy(&dst[pos], out, length);
175 :
176 1358360 : }
177 :
178 : static int
179 877935 : aes_gcm_encrypt(void *_ctx, const void *src, size_t src_size,
180 : void *dst, size_t length)
181 : {
182 877935 : struct aes_gcm_ctx *ctx = _ctx;
183 877935 : int blocks = src_size / GCM_BLOCK_SIZE;
184 877935 : int exp_blocks = blocks * GCM_BLOCK_SIZE;
185 877935 : int rest = src_size - (exp_blocks);
186 877935 : uint32_t counter;
187 :
188 877935 : if (unlikely(ctx->finished))
189 3 : return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
190 :
191 877932 : if (blocks > 0) {
192 411828 : aesni_ctr32_encrypt_blocks(src, dst,
193 : blocks,
194 411828 : ALIGN16(&ctx->expanded_key),
195 411828 : ctx->gcm.Yi.c);
196 :
197 411828 : counter = _gnutls_read_uint32(ctx->gcm.Yi.c + 12);
198 411828 : counter += blocks;
199 411828 : _gnutls_write_uint32(counter, ctx->gcm.Yi.c + 12);
200 : }
201 :
202 877932 : if (rest > 0) { /* last incomplete block */
203 689537 : ctr_encrypt_last(ctx, src, dst, exp_blocks, rest);
204 689537 : ctx->finished = 1;
205 : }
206 :
207 877932 : gcm_ghash(ctx, dst, src_size);
208 877932 : ctx->gcm.len.u[1] += src_size;
209 :
210 877932 : return 0;
211 : }
212 :
213 : static int
214 694351 : aes_gcm_decrypt(void *_ctx, const void *src, size_t src_size,
215 : void *dst, size_t dst_size)
216 : {
217 694351 : struct aes_gcm_ctx *ctx = _ctx;
218 694351 : int blocks = src_size / GCM_BLOCK_SIZE;
219 694351 : int exp_blocks = blocks * GCM_BLOCK_SIZE;
220 694351 : int rest = src_size - (exp_blocks);
221 694351 : uint32_t counter;
222 :
223 694351 : if (unlikely(ctx->finished))
224 0 : return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
225 :
226 694351 : gcm_ghash(ctx, src, src_size);
227 694351 : ctx->gcm.len.u[1] += src_size;
228 :
229 694351 : if (blocks > 0) {
230 357217 : aesni_ctr32_encrypt_blocks(src, dst,
231 : blocks,
232 357217 : ALIGN16(&ctx->expanded_key),
233 357217 : ctx->gcm.Yi.c);
234 :
235 357217 : counter = _gnutls_read_uint32(ctx->gcm.Yi.c + 12);
236 357217 : counter += blocks;
237 357217 : _gnutls_write_uint32(counter, ctx->gcm.Yi.c + 12);
238 : }
239 :
240 694351 : if (rest > 0) { /* last incomplete block */
241 668824 : ctr_encrypt_last(ctx, src, dst, exp_blocks, rest);
242 668824 : ctx->finished = 1;
243 : }
244 :
245 : return 0;
246 : }
247 :
248 1417670 : static int aes_gcm_auth(void *_ctx, const void *src, size_t src_size)
249 : {
250 1417670 : struct aes_gcm_ctx *ctx = _ctx;
251 :
252 1417670 : if (unlikely(ctx->auth_finished))
253 3 : return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
254 :
255 1417670 : gcm_ghash(ctx, src, src_size);
256 1417670 : ctx->gcm.len.u[0] += src_size;
257 :
258 1417670 : if (src_size % GCM_BLOCK_SIZE != 0)
259 1417600 : ctx->auth_finished = 1;
260 :
261 : return 0;
262 : }
263 :
264 :
265 1417670 : static void aes_gcm_tag(void *_ctx, void *tag, size_t tagsize)
266 : {
267 1417670 : struct aes_gcm_ctx *ctx = _ctx;
268 1417670 : uint8_t buffer[GCM_BLOCK_SIZE];
269 1417670 : uint64_t alen, clen;
270 :
271 1417670 : alen = ctx->gcm.len.u[0] * 8;
272 1417670 : clen = ctx->gcm.len.u[1] * 8;
273 :
274 1417670 : _gnutls_write_uint64(alen, buffer);
275 1417670 : _gnutls_write_uint64(clen, &buffer[8]);
276 :
277 1417670 : gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, buffer,
278 : GCM_BLOCK_SIZE);
279 :
280 1417670 : ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0];
281 1417670 : ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1];
282 :
283 1417670 : memcpy(tag, ctx->gcm.Xi.c, MIN(GCM_BLOCK_SIZE, tagsize));
284 1417670 : }
285 :
286 : #ifdef ASM_X86_64
287 : /* requires AVX */
288 : static int
289 369504 : aesni_gcm_aead_encrypt(void *_ctx,
290 : const void *nonce, size_t nonce_size,
291 : const void *auth, size_t auth_size,
292 : size_t tag_size,
293 : const void *plain, size_t plain_size,
294 : void *encr, size_t encr_size)
295 : {
296 369504 : struct aes_gcm_ctx *ctx = _ctx;
297 369504 : size_t s = 0;
298 :
299 369504 : if (encr_size < plain_size + tag_size)
300 0 : return gnutls_assert_val(GNUTLS_E_SHORT_MEMORY_BUFFER);
301 :
302 369504 : aes_gcm_setiv(ctx, nonce, nonce_size);
303 369504 : aes_gcm_auth(ctx, auth, auth_size);
304 :
305 369504 : if (plain_size >= 96) {
306 563172 : s = aesni_gcm_encrypt(plain, encr, plain_size, ALIGN16(&ctx->expanded_key),
307 281586 : ctx->gcm.Yi.c, ctx->gcm.Xi.u);
308 281586 : ctx->gcm.len.u[1] += s;
309 : }
310 :
311 369504 : if ((plain_size-s) > 0)
312 366647 : aes_gcm_encrypt(ctx, ((uint8_t*)plain)+s, plain_size-s, ((uint8_t*)encr)+s, encr_size-s);
313 :
314 369504 : aes_gcm_tag(ctx, ((uint8_t*)encr) + plain_size, tag_size);
315 :
316 369504 : return 0;
317 : }
318 :
319 : static int
320 698606 : aesni_gcm_aead_decrypt(void *_ctx,
321 : const void *nonce, size_t nonce_size,
322 : const void *auth, size_t auth_size,
323 : size_t tag_size,
324 : const void *encr, size_t encr_size,
325 : void *plain, size_t plain_size)
326 : {
327 698606 : struct aes_gcm_ctx *ctx = _ctx;
328 698606 : uint8_t tag[MAX_HASH_SIZE];
329 698606 : size_t s = 0;
330 :
331 698606 : if (encr_size < tag_size)
332 0 : return gnutls_assert_val(GNUTLS_E_DECRYPTION_FAILED);
333 :
334 698606 : aes_gcm_setiv(ctx, nonce, nonce_size);
335 698606 : aes_gcm_auth(ctx, auth, auth_size);
336 :
337 698606 : encr_size -= tag_size;
338 :
339 698606 : if (encr_size >= 96) {
340 840882 : s = aesni_gcm_decrypt(encr, plain, encr_size, ALIGN16(&ctx->expanded_key),
341 420441 : ctx->gcm.Yi.c, ctx->gcm.Xi.u);
342 420441 : ctx->gcm.len.u[1] += s;
343 : }
344 :
345 698606 : if ((encr_size-s) > 0) {
346 694331 : aes_gcm_decrypt(ctx, ((uint8_t*)encr)+s, encr_size-s, ((uint8_t*)plain)+s, plain_size-s);
347 : }
348 :
349 698606 : aes_gcm_tag(ctx, tag, tag_size);
350 :
351 698606 : if (gnutls_memcmp(((uint8_t*)encr)+encr_size, tag, tag_size) != 0)
352 736 : return gnutls_assert_val(GNUTLS_E_DECRYPTION_FAILED);
353 :
354 : return 0;
355 : }
356 : #else
357 : # define aesni_gcm_aead_decrypt aes_gcm_aead_decrypt
358 : # define aesni_gcm_aead_encrypt aes_gcm_aead_encrypt
359 : # include "aes-gcm-aead.h"
360 : #endif
361 :
362 : const gnutls_crypto_cipher_st _gnutls_aes_gcm_pclmul_avx = {
363 : .init = aes_gcm_cipher_init,
364 : .setkey = aes_gcm_cipher_setkey,
365 : .setiv = aes_gcm_setiv,
366 : .aead_encrypt = aesni_gcm_aead_encrypt,
367 : .aead_decrypt = aesni_gcm_aead_decrypt,
368 : .encrypt = aes_gcm_encrypt,
369 : .decrypt = aes_gcm_decrypt,
370 : .deinit = aes_gcm_deinit,
371 : .tag = aes_gcm_tag,
372 : .auth = aes_gcm_auth,
373 : };
|
