1// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build (ppc64le || ppc64) && !purego
6
7package aes
8
9import (
10	"crypto/cipher"
11	"crypto/subtle"
12	"errors"
13	"internal/byteorder"
14	"runtime"
15)
16
17// This file implements GCM using an optimized GHASH function.
18
19//go:noescape
20func gcmInit(productTable *[256]byte, h []byte)
21
22//go:noescape
23func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)
24
25//go:noescape
26func gcmMul(output []byte, productTable *[256]byte)
27
28const (
29	gcmCounterSize       = 16
30	gcmBlockSize         = 16
31	gcmTagSize           = 16
32	gcmStandardNonceSize = 12
33)
34
35var errOpen = errors.New("cipher: message authentication failed")
36
37// Assert that aesCipherGCM implements the gcmAble interface.
38var _ gcmAble = (*aesCipherAsm)(nil)
39
40type gcmAsm struct {
41	cipher *aesCipherAsm
42	// ks is the key schedule, the length of which depends on the size of
43	// the AES key.
44	ks []uint32
45	// productTable contains pre-computed multiples of the binary-field
46	// element used in GHASH.
47	productTable [256]byte
48	// nonceSize contains the expected size of the nonce, in bytes.
49	nonceSize int
50	// tagSize contains the size of the tag, in bytes.
51	tagSize int
52}
53
54func counterCryptASM(nr int, out, in []byte, counter *[gcmBlockSize]byte, key *uint32)
55
56// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only
57// called by [crypto/cipher.NewGCM] via the gcmAble interface.
58func (c *aesCipherAsm) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) {
59	var h1, h2 uint64
60	g := &gcmAsm{cipher: c, ks: c.enc[:c.l], nonceSize: nonceSize, tagSize: tagSize}
61
62	hle := make([]byte, gcmBlockSize)
63
64	c.Encrypt(hle, hle)
65
66	// Reverse the bytes in each 8 byte chunk
67	// Load little endian, store big endian
68	if runtime.GOARCH == "ppc64le" {
69		h1 = byteorder.LeUint64(hle[:8])
70		h2 = byteorder.LeUint64(hle[8:])
71	} else {
72		h1 = byteorder.BeUint64(hle[:8])
73		h2 = byteorder.BeUint64(hle[8:])
74	}
75	byteorder.BePutUint64(hle[:8], h1)
76	byteorder.BePutUint64(hle[8:], h2)
77	gcmInit(&g.productTable, hle)
78
79	return g, nil
80}
81
82func (g *gcmAsm) NonceSize() int {
83	return g.nonceSize
84}
85
86func (g *gcmAsm) Overhead() int {
87	return g.tagSize
88}
89
90func sliceForAppend(in []byte, n int) (head, tail []byte) {
91	if total := len(in) + n; cap(in) >= total {
92		head = in[:total]
93	} else {
94		head = make([]byte, total)
95		copy(head, in)
96	}
97	tail = head[len(in):]
98	return
99}
100
101// deriveCounter computes the initial GCM counter state from the given nonce.
102func (g *gcmAsm) deriveCounter(counter *[gcmBlockSize]byte, nonce []byte) {
103	if len(nonce) == gcmStandardNonceSize {
104		copy(counter[:], nonce)
105		counter[gcmBlockSize-1] = 1
106	} else {
107		var hash [16]byte
108		g.paddedGHASH(&hash, nonce)
109		lens := gcmLengths(0, uint64(len(nonce))*8)
110		g.paddedGHASH(&hash, lens[:])
111		copy(counter[:], hash[:])
112	}
113}
114
115// counterCrypt encrypts in using AES in counter mode and places the result
116// into out. counter is the initial count value and will be updated with the next
117// count value. The length of out must be greater than or equal to the length
118// of in.
119// counterCryptASM implements counterCrypt which then allows the loop to
120// be unrolled and optimized.
121func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte) {
122	counterCryptASM(int(g.cipher.l)/4-1, out, in, counter, &g.cipher.enc[0])
123
124}
125
126// increments the rightmost 32-bits of the count value by 1.
127func gcmInc32(counterBlock *[16]byte) {
128	c := counterBlock[len(counterBlock)-4:]
129	x := byteorder.BeUint32(c) + 1
130	byteorder.BePutUint32(c, x)
131}
132
133// paddedGHASH pads data with zeroes until its length is a multiple of
134// 16-bytes. It then calculates a new value for hash using the ghash
135// algorithm.
136func (g *gcmAsm) paddedGHASH(hash *[16]byte, data []byte) {
137	if siz := len(data) - (len(data) % gcmBlockSize); siz > 0 {
138		gcmHash(hash[:], &g.productTable, data[:], siz)
139		data = data[siz:]
140	}
141	if len(data) > 0 {
142		var s [16]byte
143		copy(s[:], data)
144		gcmHash(hash[:], &g.productTable, s[:], len(s))
145	}
146}
147
148// auth calculates GHASH(ciphertext, additionalData), masks the result with
149// tagMask and writes the result to out.
150func (g *gcmAsm) auth(out, ciphertext, aad []byte, tagMask *[gcmTagSize]byte) {
151	var hash [16]byte
152	g.paddedGHASH(&hash, aad)
153	g.paddedGHASH(&hash, ciphertext)
154	lens := gcmLengths(uint64(len(aad))*8, uint64(len(ciphertext))*8)
155	g.paddedGHASH(&hash, lens[:])
156
157	copy(out, hash[:])
158	for i := range out {
159		out[i] ^= tagMask[i]
160	}
161}
162
163// Seal encrypts and authenticates plaintext. See the [cipher.AEAD] interface for
164// details.
165func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte {
166	if len(nonce) != g.nonceSize {
167		panic("cipher: incorrect nonce length given to GCM")
168	}
169	if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize {
170		panic("cipher: message too large for GCM")
171	}
172
173	ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize)
174
175	var counter, tagMask [gcmBlockSize]byte
176	g.deriveCounter(&counter, nonce)
177
178	g.cipher.Encrypt(tagMask[:], counter[:])
179	gcmInc32(&counter)
180
181	g.counterCrypt(out, plaintext, &counter)
182	g.auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask)
183
184	return ret
185}
186
187// Open authenticates and decrypts ciphertext. See the [cipher.AEAD] interface
188// for details.
189func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
190	if len(nonce) != g.nonceSize {
191		panic("cipher: incorrect nonce length given to GCM")
192	}
193	if len(ciphertext) < g.tagSize {
194		return nil, errOpen
195	}
196	if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) {
197		return nil, errOpen
198	}
199
200	tag := ciphertext[len(ciphertext)-g.tagSize:]
201	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
202
203	var counter, tagMask [gcmBlockSize]byte
204	g.deriveCounter(&counter, nonce)
205
206	g.cipher.Encrypt(tagMask[:], counter[:])
207	gcmInc32(&counter)
208
209	var expectedTag [gcmTagSize]byte
210	g.auth(expectedTag[:], ciphertext, data, &tagMask)
211
212	ret, out := sliceForAppend(dst, len(ciphertext))
213
214	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
215		clear(out)
216		return nil, errOpen
217	}
218
219	g.counterCrypt(out, ciphertext, &counter)
220	return ret, nil
221}
222
223func gcmLengths(len0, len1 uint64) [16]byte {
224	return [16]byte{
225		byte(len0 >> 56),
226		byte(len0 >> 48),
227		byte(len0 >> 40),
228		byte(len0 >> 32),
229		byte(len0 >> 24),
230		byte(len0 >> 16),
231		byte(len0 >> 8),
232		byte(len0),
233		byte(len1 >> 56),
234		byte(len1 >> 48),
235		byte(len1 >> 40),
236		byte(len1 >> 32),
237		byte(len1 >> 24),
238		byte(len1 >> 16),
239		byte(len1 >> 8),
240		byte(len1),
241	}
242}
243