1 /* inffast.c -- fast decoding
2 * Copyright (C) 1995-2017 Mark Adler
3 * For conditions of distribution and use, see copyright notice in zlib.h
4 */
5
6 #include "zbuild.h"
7 #include "zutil.h"
8 #include "inftrees.h"
9 #include "inflate.h"
10 #include "inffast.h"
11 #include "inflate_p.h"
12 #include "functable.h"
13
14 /* Load 64 bits from IN and place the bytes at offset BITS in the result. */
load_64_bits(const unsigned char * in,unsigned bits)15 static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) {
16 uint64_t chunk;
17 zmemcpy_8(&chunk, in);
18
19 #if BYTE_ORDER == LITTLE_ENDIAN
20 return chunk << bits;
21 #else
22 return ZSWAP64(chunk) << bits;
23 #endif
24 }
25 /*
26 Decode literal, length, and distance codes and write out the resulting
27 literal and match bytes until either not enough input or output is
28 available, an end-of-block is encountered, or a data error is encountered.
29 When large enough input and output buffers are supplied to inflate(), for
30 example, a 16K input buffer and a 64K output buffer, more than 95% of the
31 inflate execution time is spent in this routine.
32
33 Entry assumptions:
34
35 state->mode == LEN
36 strm->avail_in >= INFLATE_FAST_MIN_HAVE
37 strm->avail_out >= INFLATE_FAST_MIN_LEFT
38 start >= strm->avail_out
39 state->bits < 8
40
41 On return, state->mode is one of:
42
43 LEN -- ran out of enough output space or enough available input
44 TYPE -- reached end of block code, inflate() to interpret next block
45 BAD -- error in block data
46
47 Notes:
48
49 - The maximum input bits used by a length/distance pair is 15 bits for the
50 length code, 5 bits for the length extra, 15 bits for the distance code,
51 and 13 bits for the distance extra. This totals 48 bits, or six bytes.
52 Therefore if strm->avail_in >= 6, then there is enough input to avoid
53 checking for available input while decoding.
54
55 - On some architectures, it can be significantly faster (e.g. up to 1.2x
56 faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
57 time, so INFLATE_FAST_MIN_HAVE == 8.
58
59 - The maximum bytes that a single length/distance pair can output is 258
60 bytes, which is the maximum length that can be coded. inflate_fast()
61 requires strm->avail_out >= 258 for each loop to avoid checking for
62 output space.
63 */
zng_inflate_fast(PREFIX3 (stream)* strm,unsigned long start)64 void Z_INTERNAL zng_inflate_fast(PREFIX3(stream) *strm, unsigned long start) {
65 /* start: inflate()'s starting value for strm->avail_out */
66 struct inflate_state *state;
67 z_const unsigned char *in; /* local strm->next_in */
68 const unsigned char *last; /* have enough input while in < last */
69 unsigned char *out; /* local strm->next_out */
70 unsigned char *beg; /* inflate()'s initial strm->next_out */
71 unsigned char *end; /* while out < end, enough space available */
72 unsigned char *safe; /* can use chunkcopy provided out < safe */
73 #ifdef INFLATE_STRICT
74 unsigned dmax; /* maximum distance from zlib header */
75 #endif
76 unsigned wsize; /* window size or zero if not using window */
77 unsigned whave; /* valid bytes in the window */
78 unsigned wnext; /* window write index */
79 unsigned char *window; /* allocated sliding window, if wsize != 0 */
80
81 /* hold is a local copy of strm->hold. By default, hold satisfies the same
82 invariants that strm->hold does, namely that (hold >> bits) == 0. This
83 invariant is kept by loading bits into hold one byte at a time, like:
84
85 hold |= next_byte_of_input << bits; in++; bits += 8;
86
87 If we need to ensure that bits >= 15 then this code snippet is simply
88 repeated. Over one iteration of the outermost do/while loop, this
89 happens up to six times (48 bits of input), as described in the NOTES
90 above.
91
92 However, on some little endian architectures, it can be significantly
93 faster to load 64 bits once instead of 8 bits six times:
94
95 if (bits <= 16) {
96 hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
97 }
98
99 Unlike the simpler one byte load, shifting the next_8_bytes_of_input
100 by bits will overflow and lose those high bits, up to 2 bytes' worth.
101 The conservative estimate is therefore that we have read only 6 bytes
102 (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
103 rest of the iteration, and we will not need to load another 8 bytes.
104
105 Inside this function, we no longer satisfy (hold >> bits) == 0, but
106 this is not problematic, even if that overflow does not land on an 8 bit
107 byte boundary. Those excess bits will eventually shift down lower as the
108 Huffman decoder consumes input, and when new input bits need to be loaded
109 into the bits variable, the same input bits will be or'ed over those
110 existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
111 Note that we therefore write that load operation as "hold |= etc" and not
112 "hold += etc".
113
114 Outside that loop, at the end of the function, hold is bitwise and'ed
115 with (1<<bits)-1 to drop those excess bits so that, on function exit, we
116 keep the invariant that (state->hold >> state->bits) == 0.
117 */
118 uint64_t hold; /* local strm->hold */
119 unsigned bits; /* local strm->bits */
120 code const *lcode; /* local strm->lencode */
121 code const *dcode; /* local strm->distcode */
122 unsigned lmask; /* mask for first level of length codes */
123 unsigned dmask; /* mask for first level of distance codes */
124 const code *here; /* retrieved table entry */
125 unsigned op; /* code bits, operation, extra bits, or */
126 /* window position, window bytes to copy */
127 unsigned len; /* match length, unused bytes */
128 unsigned dist; /* match distance */
129 unsigned char *from; /* where to copy match from */
130 unsigned extra_safe; /* copy chunks safely in all cases */
131
132 /* copy state to local variables */
133 state = (struct inflate_state *)strm->state;
134 in = strm->next_in;
135 last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
136 out = strm->next_out;
137 beg = out - (start - strm->avail_out);
138 end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
139 safe = out + strm->avail_out;
140 #ifdef INFLATE_STRICT
141 dmax = state->dmax;
142 #endif
143 wsize = state->wsize;
144 whave = state->whave;
145 wnext = state->wnext;
146 window = state->window;
147 hold = state->hold;
148 bits = state->bits;
149 lcode = state->lencode;
150 dcode = state->distcode;
151 lmask = (1U << state->lenbits) - 1;
152 dmask = (1U << state->distbits) - 1;
153
154 /* Detect if out and window point to the same memory allocation. In this instance it is
155 necessary to use safe chunk copy functions to prevent overwriting the window. If the
156 window is overwritten then future matches with far distances will fail to copy correctly. */
157 extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + wsize);
158
159 /* decode literals and length/distances until end-of-block or not enough
160 input data or output space */
161 do {
162 if (bits < 15) {
163 hold |= load_64_bits(in, bits);
164 in += 6;
165 bits += 48;
166 }
167 here = lcode + (hold & lmask);
168 dolen:
169 DROPBITS(here->bits);
170 op = here->op;
171 if (op == 0) { /* literal */
172 Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
173 "inflate: literal '%c'\n" :
174 "inflate: literal 0x%02x\n", here->val));
175 *out++ = (unsigned char)(here->val);
176 } else if (op & 16) { /* length base */
177 len = here->val;
178 op &= 15; /* number of extra bits */
179 if (bits < op) {
180 hold |= load_64_bits(in, bits);
181 in += 6;
182 bits += 48;
183 }
184 len += BITS(op);
185 DROPBITS(op);
186 Tracevv((stderr, "inflate: length %u\n", len));
187 if (bits < 15) {
188 hold |= load_64_bits(in, bits);
189 in += 6;
190 bits += 48;
191 }
192 here = dcode + (hold & dmask);
193 dodist:
194 DROPBITS(here->bits);
195 op = here->op;
196 if (op & 16) { /* distance base */
197 dist = here->val;
198 op &= 15; /* number of extra bits */
199 if (bits < op) {
200 hold |= load_64_bits(in, bits);
201 in += 6;
202 bits += 48;
203 }
204 dist += BITS(op);
205 #ifdef INFLATE_STRICT
206 if (dist > dmax) {
207 SET_BAD("invalid distance too far back");
208 break;
209 }
210 #endif
211 DROPBITS(op);
212 Tracevv((stderr, "inflate: distance %u\n", dist));
213 op = (unsigned)(out - beg); /* max distance in output */
214 if (dist > op) { /* see if copy from window */
215 op = dist - op; /* distance back in window */
216 if (op > whave) {
217 if (state->sane) {
218 SET_BAD("invalid distance too far back");
219 break;
220 }
221 #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
222 if (len <= op - whave) {
223 do {
224 *out++ = 0;
225 } while (--len);
226 continue;
227 }
228 len -= op - whave;
229 do {
230 *out++ = 0;
231 } while (--op > whave);
232 if (op == 0) {
233 from = out - dist;
234 do {
235 *out++ = *from++;
236 } while (--len);
237 continue;
238 }
239 #endif
240 }
241 from = window;
242 if (wnext == 0) { /* very common case */
243 from += wsize - op;
244 } else if (wnext >= op) { /* contiguous in window */
245 from += wnext - op;
246 } else { /* wrap around window */
247 op -= wnext;
248 from += wsize - op;
249 if (op < len) { /* some from end of window */
250 len -= op;
251 out = chunkcopy_safe(out, from, op, safe);
252 from = window; /* more from start of window */
253 op = wnext;
254 /* This (rare) case can create a situation where
255 the first chunkcopy below must be checked.
256 */
257 }
258 }
259 if (op < len) { /* still need some from output */
260 len -= op;
261 out = chunkcopy_safe(out, from, op, safe);
262 out = functable.chunkunroll(out, &dist, &len);
263 out = chunkcopy_safe(out, out - dist, len, safe);
264 } else {
265 out = chunkcopy_safe(out, from, len, safe);
266 }
267 } else if (extra_safe) {
268 /* Whole reference is in range of current output. */
269 if (dist >= len || dist >= state->chunksize)
270 out = chunkcopy_safe(out, out - dist, len, safe);
271 else
272 out = functable.chunkmemset_safe(out, dist, len, (unsigned)((safe - out) + 1));
273 } else {
274 /* Whole reference is in range of current output. No range checks are
275 necessary because we start with room for at least 258 bytes of output,
276 so unroll and roundoff operations can write beyond `out+len` so long
277 as they stay within 258 bytes of `out`.
278 */
279 if (dist >= len || dist >= state->chunksize)
280 out = functable.chunkcopy(out, out - dist, len);
281 else
282 out = functable.chunkmemset(out, dist, len);
283 }
284 } else if ((op & 64) == 0) { /* 2nd level distance code */
285 here = dcode + here->val + BITS(op);
286 goto dodist;
287 } else {
288 SET_BAD("invalid distance code");
289 break;
290 }
291 } else if ((op & 64) == 0) { /* 2nd level length code */
292 here = lcode + here->val + BITS(op);
293 goto dolen;
294 } else if (op & 32) { /* end-of-block */
295 Tracevv((stderr, "inflate: end of block\n"));
296 state->mode = TYPE;
297 break;
298 } else {
299 SET_BAD("invalid literal/length code");
300 break;
301 }
302 } while (in < last && out < end);
303
304 /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
305 len = bits >> 3;
306 in -= len;
307 bits -= len << 3;
308 hold &= (UINT64_C(1) << bits) - 1;
309
310 /* update state and return */
311 strm->next_in = in;
312 strm->next_out = out;
313 strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
314 : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
315 strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
316 : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
317
318 Assert(bits <= 32, "Remaining bits greater than 32");
319 state->hold = (uint32_t)hold;
320 state->bits = bits;
321 return;
322 }
323
324 /*
325 inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
326 - Using bit fields for code structure
327 - Different op definition to avoid & for extra bits (do & for table bits)
328 - Three separate decoding do-loops for direct, window, and wnext == 0
329 - Special case for distance > 1 copies to do overlapped load and store copy
330 - Explicit branch predictions (based on measured branch probabilities)
331 - Deferring match copy and interspersed it with decoding subsequent codes
332 - Swapping literal/length else
333 - Swapping window/direct else
334 - Larger unrolled copy loops (three is about right)
335 - Moving len -= 3 statement into middle of loop
336 */
337