1// Copyright 2016 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5//go:build !purego 6 7package aes 8 9import ( 10 "crypto/cipher" 11 "crypto/internal/alias" 12 "crypto/subtle" 13 "errors" 14 "internal/byteorder" 15 "internal/cpu" 16) 17 18// This file contains two implementations of AES-GCM. The first implementation 19// (gcmAsm) uses the KMCTR instruction to encrypt using AES in counter mode and 20// the KIMD instruction for GHASH. The second implementation (gcmKMA) uses the 21// newer KMA instruction which performs both operations. 22 23// gcmCount represents a 16-byte big-endian count value. 24type gcmCount [16]byte 25 26// inc increments the rightmost 32-bits of the count value by 1. 27func (x *gcmCount) inc() { 28 byteorder.BePutUint32(x[len(x)-4:], byteorder.BeUint32(x[len(x)-4:])+1) 29} 30 31// gcmLengths writes len0 || len1 as big-endian values to a 16-byte array. 32func gcmLengths(len0, len1 uint64) [16]byte { 33 v := [16]byte{} 34 byteorder.BePutUint64(v[0:], len0) 35 byteorder.BePutUint64(v[8:], len1) 36 return v 37} 38 39// gcmHashKey represents the 16-byte hash key required by the GHASH algorithm. 40type gcmHashKey [16]byte 41 42type gcmAsm struct { 43 block *aesCipherAsm 44 hashKey gcmHashKey 45 nonceSize int 46 tagSize int 47} 48 49const ( 50 gcmBlockSize = 16 51 gcmTagSize = 16 52 gcmMinimumTagSize = 12 // NIST SP 800-38D recommends tags with 12 or more bytes. 53 gcmStandardNonceSize = 12 54) 55 56var errOpen = errors.New("cipher: message authentication failed") 57 58// Assert that aesCipherAsm implements the gcmAble interface. 59var _ gcmAble = (*aesCipherAsm)(nil) 60 61// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only 62// called by [crypto/cipher.NewGCM] via the gcmAble interface. 63func (c *aesCipherAsm) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) { 64 var hk gcmHashKey 65 c.Encrypt(hk[:], hk[:]) 66 g := gcmAsm{ 67 block: c, 68 hashKey: hk, 69 nonceSize: nonceSize, 70 tagSize: tagSize, 71 } 72 if cpu.S390X.HasAESGCM { 73 g := gcmKMA{g} 74 return &g, nil 75 } 76 return &g, nil 77} 78 79func (g *gcmAsm) NonceSize() int { 80 return g.nonceSize 81} 82 83func (g *gcmAsm) Overhead() int { 84 return g.tagSize 85} 86 87// sliceForAppend takes a slice and a requested number of bytes. It returns a 88// slice with the contents of the given slice followed by that many bytes and a 89// second slice that aliases into it and contains only the extra bytes. If the 90// original slice has sufficient capacity then no allocation is performed. 91func sliceForAppend(in []byte, n int) (head, tail []byte) { 92 if total := len(in) + n; cap(in) >= total { 93 head = in[:total] 94 } else { 95 head = make([]byte, total) 96 copy(head, in) 97 } 98 tail = head[len(in):] 99 return 100} 101 102// ghash uses the GHASH algorithm to hash data with the given key. The initial 103// hash value is given by hash which will be updated with the new hash value. 104// The length of data must be a multiple of 16-bytes. 105// 106//go:noescape 107func ghash(key *gcmHashKey, hash *[16]byte, data []byte) 108 109// paddedGHASH pads data with zeroes until its length is a multiple of 110// 16-bytes. It then calculates a new value for hash using the GHASH algorithm. 111func (g *gcmAsm) paddedGHASH(hash *[16]byte, data []byte) { 112 siz := len(data) &^ 0xf // align size to 16-bytes 113 if siz > 0 { 114 ghash(&g.hashKey, hash, data[:siz]) 115 data = data[siz:] 116 } 117 if len(data) > 0 { 118 var s [16]byte 119 copy(s[:], data) 120 ghash(&g.hashKey, hash, s[:]) 121 } 122} 123 124// cryptBlocksGCM encrypts src using AES in counter mode using the given 125// function code and key. The rightmost 32-bits of the counter are incremented 126// between each block as required by the GCM spec. The initial counter value 127// is given by cnt, which is updated with the value of the next counter value 128// to use. 129// 130// The lengths of both dst and buf must be greater than or equal to the length 131// of src. buf may be partially or completely overwritten during the execution 132// of the function. 133// 134//go:noescape 135func cryptBlocksGCM(fn code, key, dst, src, buf []byte, cnt *gcmCount) 136 137// counterCrypt encrypts src using AES in counter mode and places the result 138// into dst. cnt is the initial count value and will be updated with the next 139// count value. The length of dst must be greater than or equal to the length 140// of src. 141func (g *gcmAsm) counterCrypt(dst, src []byte, cnt *gcmCount) { 142 // Copying src into a buffer improves performance on some models when 143 // src and dst point to the same underlying array. We also need a 144 // buffer for counter values. 145 var ctrbuf, srcbuf [2048]byte 146 for len(src) >= 16 { 147 siz := len(src) 148 if len(src) > len(ctrbuf) { 149 siz = len(ctrbuf) 150 } 151 siz &^= 0xf // align siz to 16-bytes 152 copy(srcbuf[:], src[:siz]) 153 cryptBlocksGCM(g.block.function, g.block.key, dst[:siz], srcbuf[:siz], ctrbuf[:], cnt) 154 src = src[siz:] 155 dst = dst[siz:] 156 } 157 if len(src) > 0 { 158 var x [16]byte 159 g.block.Encrypt(x[:], cnt[:]) 160 for i := range src { 161 dst[i] = src[i] ^ x[i] 162 } 163 cnt.inc() 164 } 165} 166 167// deriveCounter computes the initial GCM counter state from the given nonce. 168// See NIST SP 800-38D, section 7.1. 169func (g *gcmAsm) deriveCounter(nonce []byte) gcmCount { 170 // GCM has two modes of operation with respect to the initial counter 171 // state: a "fast path" for 96-bit (12-byte) nonces, and a "slow path" 172 // for nonces of other lengths. For a 96-bit nonce, the nonce, along 173 // with a four-byte big-endian counter starting at one, is used 174 // directly as the starting counter. For other nonce sizes, the counter 175 // is computed by passing it through the GHASH function. 176 var counter gcmCount 177 if len(nonce) == gcmStandardNonceSize { 178 copy(counter[:], nonce) 179 counter[gcmBlockSize-1] = 1 180 } else { 181 var hash [16]byte 182 g.paddedGHASH(&hash, nonce) 183 lens := gcmLengths(0, uint64(len(nonce))*8) 184 g.paddedGHASH(&hash, lens[:]) 185 copy(counter[:], hash[:]) 186 } 187 return counter 188} 189 190// auth calculates GHASH(ciphertext, additionalData), masks the result with 191// tagMask and writes the result to out. 192func (g *gcmAsm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize]byte) { 193 var hash [16]byte 194 g.paddedGHASH(&hash, additionalData) 195 g.paddedGHASH(&hash, ciphertext) 196 lens := gcmLengths(uint64(len(additionalData))*8, uint64(len(ciphertext))*8) 197 g.paddedGHASH(&hash, lens[:]) 198 199 copy(out, hash[:]) 200 for i := range out { 201 out[i] ^= tagMask[i] 202 } 203} 204 205// Seal encrypts and authenticates plaintext. See the [cipher.AEAD] interface for 206// details. 207func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte { 208 if len(nonce) != g.nonceSize { 209 panic("crypto/cipher: incorrect nonce length given to GCM") 210 } 211 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 212 panic("crypto/cipher: message too large for GCM") 213 } 214 215 ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize) 216 if alias.InexactOverlap(out[:len(plaintext)], plaintext) { 217 panic("crypto/cipher: invalid buffer overlap") 218 } 219 220 counter := g.deriveCounter(nonce) 221 222 var tagMask [gcmBlockSize]byte 223 g.block.Encrypt(tagMask[:], counter[:]) 224 counter.inc() 225 226 var tagOut [gcmTagSize]byte 227 g.counterCrypt(out, plaintext, &counter) 228 g.auth(tagOut[:], out[:len(plaintext)], data, &tagMask) 229 copy(out[len(plaintext):], tagOut[:]) 230 231 return ret 232} 233 234// Open authenticates and decrypts ciphertext. See the [cipher.AEAD] interface 235// for details. 236func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 237 if len(nonce) != g.nonceSize { 238 panic("crypto/cipher: incorrect nonce length given to GCM") 239 } 240 // Sanity check to prevent the authentication from always succeeding if an implementation 241 // leaves tagSize uninitialized, for example. 242 if g.tagSize < gcmMinimumTagSize { 243 panic("crypto/cipher: incorrect GCM tag size") 244 } 245 if len(ciphertext) < g.tagSize { 246 return nil, errOpen 247 } 248 if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) { 249 return nil, errOpen 250 } 251 252 tag := ciphertext[len(ciphertext)-g.tagSize:] 253 ciphertext = ciphertext[:len(ciphertext)-g.tagSize] 254 255 counter := g.deriveCounter(nonce) 256 257 var tagMask [gcmBlockSize]byte 258 g.block.Encrypt(tagMask[:], counter[:]) 259 counter.inc() 260 261 var expectedTag [gcmTagSize]byte 262 g.auth(expectedTag[:], ciphertext, data, &tagMask) 263 264 ret, out := sliceForAppend(dst, len(ciphertext)) 265 if alias.InexactOverlap(out, ciphertext) { 266 panic("crypto/cipher: invalid buffer overlap") 267 } 268 269 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 { 270 // The AESNI code decrypts and authenticates concurrently, and 271 // so overwrites dst in the event of a tag mismatch. That 272 // behavior is mimicked here in order to be consistent across 273 // platforms. 274 clear(out) 275 return nil, errOpen 276 } 277 278 g.counterCrypt(out, ciphertext, &counter) 279 return ret, nil 280} 281 282// gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should 283// only be used if hasKMA is true. 284type gcmKMA struct { 285 gcmAsm 286} 287 288// flags for the KMA instruction 289const ( 290 kmaHS = 1 << 10 // hash subkey supplied 291 kmaLAAD = 1 << 9 // last series of additional authenticated data 292 kmaLPC = 1 << 8 // last series of plaintext or ciphertext blocks 293 kmaDecrypt = 1 << 7 // decrypt 294) 295 296// kmaGCM executes the encryption or decryption operation given by fn. The tag 297// will be calculated and written to tag. cnt should contain the current 298// counter state and will be overwritten with the updated counter state. 299// TODO(mundaym): could pass in hash subkey 300// 301//go:noescape 302func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount) 303 304// Seal encrypts and authenticates plaintext. See the [cipher.AEAD] interface for 305// details. 306func (g *gcmKMA) Seal(dst, nonce, plaintext, data []byte) []byte { 307 if len(nonce) != g.nonceSize { 308 panic("crypto/cipher: incorrect nonce length given to GCM") 309 } 310 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 311 panic("crypto/cipher: message too large for GCM") 312 } 313 314 ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize) 315 if alias.InexactOverlap(out[:len(plaintext)], plaintext) { 316 panic("crypto/cipher: invalid buffer overlap") 317 } 318 319 counter := g.deriveCounter(nonce) 320 fc := g.block.function | kmaLAAD | kmaLPC 321 322 var tag [gcmTagSize]byte 323 kmaGCM(fc, g.block.key, out[:len(plaintext)], plaintext, data, &tag, &counter) 324 copy(out[len(plaintext):], tag[:]) 325 326 return ret 327} 328 329// Open authenticates and decrypts ciphertext. See the [cipher.AEAD] interface 330// for details. 331func (g *gcmKMA) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 332 if len(nonce) != g.nonceSize { 333 panic("crypto/cipher: incorrect nonce length given to GCM") 334 } 335 if len(ciphertext) < g.tagSize { 336 return nil, errOpen 337 } 338 if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) { 339 return nil, errOpen 340 } 341 342 tag := ciphertext[len(ciphertext)-g.tagSize:] 343 ciphertext = ciphertext[:len(ciphertext)-g.tagSize] 344 ret, out := sliceForAppend(dst, len(ciphertext)) 345 if alias.InexactOverlap(out, ciphertext) { 346 panic("crypto/cipher: invalid buffer overlap") 347 } 348 349 if g.tagSize < gcmMinimumTagSize { 350 panic("crypto/cipher: incorrect GCM tag size") 351 } 352 353 counter := g.deriveCounter(nonce) 354 fc := g.block.function | kmaLAAD | kmaLPC | kmaDecrypt 355 356 var expectedTag [gcmTagSize]byte 357 kmaGCM(fc, g.block.key, out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter) 358 359 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 { 360 // The AESNI code decrypts and authenticates concurrently, and 361 // so overwrites dst in the event of a tag mismatch. That 362 // behavior is mimicked here in order to be consistent across 363 // platforms. 364 clear(out) 365 return nil, errOpen 366 } 367 368 return ret, nil 369} 370