1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
/*
* Accelerated poly_hash implementation with ARMv8 PMULL instructions.
*
* Based on ghash-ce-glue.c.
*
* poly_hash is part of the HEH (Hash-Encrypt-Hash) encryption mode, proposed in
* Internet Draft https://tools.ietf.org/html/draft-cope-heh-01.
*
* poly_hash is very similar to GHASH: both algorithms are keyed hashes which
* interpret their input data as coefficients of a polynomial over GF(2^128),
* then calculate a hash value by evaluating that polynomial at the point given
* by the key, e.g. using Horner's rule. The difference is that poly_hash uses
* the more natural "ble" convention to represent GF(2^128) elements, whereas
* GHASH uses the less natural "lle" convention (see include/crypto/gf128mul.h).
* The ble convention makes it simpler to implement GF(2^128) multiplication.
*
* Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
* Copyright (C) 2017 Google Inc. <ebiggers@google.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation.
*/
#include <asm/neon.h>
#include <crypto/b128ops.h>
#include <crypto/internal/hash.h>
#include <linux/cpufeature.h>
#include <linux/crypto.h>
#include <linux/module.h>
/*
* Note: in this algorithm we currently use 'le128' to represent GF(2^128)
* elements, even though poly_hash-generic uses 'be128'. Both types are
* actually "wrong" because the elements are actually in 'ble' format, and there
* should be a ble type to represent this --- as well as lle, bbe, and lbe types
* for the other conventions for representing GF(2^128) elements. But
* practically it doesn't matter which type we choose here, so we just use le128
* since it's arguably more accurate, while poly_hash-generic still has to use
* be128 because the generic GF(2^128) multiplication functions all take be128.
*/
struct poly_hash_desc_ctx {
le128 digest;
unsigned int count;
};
asmlinkage void pmull_poly_hash_update(le128 *digest, const le128 *key,
const u8 *src, unsigned int blocks,
unsigned int partial);
static int poly_hash_setkey(struct crypto_shash *tfm,
const u8 *key, unsigned int keylen)
{
if (keylen != sizeof(le128)) {
crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(crypto_shash_ctx(tfm), key, sizeof(le128));
return 0;
}
static int poly_hash_init(struct shash_desc *desc)
{
struct poly_hash_desc_ctx *ctx = shash_desc_ctx(desc);
ctx->digest = (le128) { 0 };
ctx->count = 0;
return 0;
}
static int poly_hash_update(struct shash_desc *desc, const u8 *src,
unsigned int len)
{
struct poly_hash_desc_ctx *ctx = shash_desc_ctx(desc);
unsigned int partial = ctx->count % sizeof(le128);
u8 *dst = (u8 *)&ctx->digest + partial;
ctx->count += len;
/* Finishing at least one block? */
if (partial + len >= sizeof(le128)) {
const le128 *key = crypto_shash_ctx(desc->tfm);
if (partial) {
/* Finish the pending block. */
unsigned int n = sizeof(le128) - partial;
len -= n;
do {
*dst++ ^= *src++;
} while (--n);
}
/*
* Do the real work. If 'partial' is nonzero, this starts by
* multiplying 'digest' by 'key'. Then for each additional full
* block it adds the block to 'digest' and multiplies by 'key'.
*/
kernel_neon_begin_partial(8);
pmull_poly_hash_update(&ctx->digest, key, src,
len / sizeof(le128), partial);
kernel_neon_end();
src += len - (len % sizeof(le128));
len %= sizeof(le128);
dst = (u8 *)&ctx->digest;
}
/* Continue adding the next block to 'digest'. */
while (len--)
*dst++ ^= *src++;
return 0;
}
static int poly_hash_final(struct shash_desc *desc, u8 *out)
{
struct poly_hash_desc_ctx *ctx = shash_desc_ctx(desc);
unsigned int partial = ctx->count % sizeof(le128);
/* Finish the last block if needed. */
if (partial) {
const le128 *key = crypto_shash_ctx(desc->tfm);
kernel_neon_begin_partial(8);
pmull_poly_hash_update(&ctx->digest, key, NULL, 0, partial);
kernel_neon_end();
}
memcpy(out, &ctx->digest, sizeof(le128));
return 0;
}
static struct shash_alg poly_hash_alg = {
.digestsize = sizeof(le128),
.init = poly_hash_init,
.update = poly_hash_update,
.final = poly_hash_final,
.setkey = poly_hash_setkey,
.descsize = sizeof(struct poly_hash_desc_ctx),
.base = {
.cra_name = "poly_hash",
.cra_driver_name = "poly_hash-ce",
.cra_priority = 300,
.cra_ctxsize = sizeof(le128),
.cra_module = THIS_MODULE,
},
};
static int __init poly_hash_ce_mod_init(void)
{
return crypto_register_shash(&poly_hash_alg);
}
static void __exit poly_hash_ce_mod_exit(void)
{
crypto_unregister_shash(&poly_hash_alg);
}
MODULE_DESCRIPTION("Polynomial evaluation hash using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
MODULE_LICENSE("GPL v2");
module_cpu_feature_match(PMULL, poly_hash_ce_mod_init);
module_exit(poly_hash_ce_mod_exit);
|
