1 // Copyright (c) 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 // Decoder to test QPACK Offline Interop corpus
6 //
7 // See https://github.com/quicwg/base-drafts/wiki/QPACK-Offline-Interop for
8 // description of test data format.
9 //
10 // Example usage
11 //
12 // cd $TEST_DATA
13 // git clone https://github.com/qpackers/qifs.git
14 // TEST_ENCODED_DATA=$TEST_DATA/qifs/encoded/qpack-06
15 // TEST_QIF_DATA=$TEST_DATA/qifs/qifs
16 // $BIN/qpack_offline_decoder \
17 // $TEST_ENCODED_DATA/f5/fb-req.qifencoded.4096.100.0 \
18 // $TEST_QIF_DATA/fb-req.qif
19 // $TEST_ENCODED_DATA/h2o/fb-req-hq.out.512.0.1 \
20 // $TEST_QIF_DATA/fb-req-hq.qif
21 // $TEST_ENCODED_DATA/ls-qpack/fb-resp-hq.out.0.0.0 \
22 // $TEST_QIF_DATA/fb-resp-hq.qif
23 // $TEST_ENCODED_DATA/proxygen/netbsd.qif.proxygen.out.4096.0.0 \
24 // $TEST_QIF_DATA/netbsd.qif
25 //
26
27 #include "quiche/quic/test_tools/qpack/qpack_offline_decoder.h"
28
29 #include <cstdint>
30 #include <string>
31 #include <utility>
32
33 #include "absl/strings/match.h"
34 #include "absl/strings/numbers.h"
35 #include "absl/strings/str_split.h"
36 #include "absl/strings/string_view.h"
37 #include "quiche/quic/core/quic_types.h"
38 #include "quiche/quic/platform/api/quic_logging.h"
39 #include "quiche/quic/test_tools/qpack/qpack_test_utils.h"
40 #include "quiche/common/platform/api/quiche_file_utils.h"
41 #include "quiche/common/quiche_endian.h"
42
43 namespace quic {
44
QpackOfflineDecoder()45 QpackOfflineDecoder::QpackOfflineDecoder()
46 : encoder_stream_error_detected_(false) {}
47
DecodeAndVerifyOfflineData(absl::string_view input_filename,absl::string_view expected_headers_filename)48 bool QpackOfflineDecoder::DecodeAndVerifyOfflineData(
49 absl::string_view input_filename,
50 absl::string_view expected_headers_filename) {
51 if (!ParseInputFilename(input_filename)) {
52 QUIC_LOG(ERROR) << "Error parsing input filename " << input_filename;
53 return false;
54 }
55
56 if (!DecodeHeaderBlocksFromFile(input_filename)) {
57 QUIC_LOG(ERROR) << "Error decoding header blocks in " << input_filename;
58 return false;
59 }
60
61 if (!VerifyDecodedHeaderLists(expected_headers_filename)) {
62 QUIC_LOG(ERROR) << "Header lists decoded from " << input_filename
63 << " to not match expected headers parsed from "
64 << expected_headers_filename;
65 return false;
66 }
67
68 return true;
69 }
70
OnEncoderStreamError(QuicErrorCode error_code,absl::string_view error_message)71 void QpackOfflineDecoder::OnEncoderStreamError(
72 QuicErrorCode error_code, absl::string_view error_message) {
73 QUIC_LOG(ERROR) << "Encoder stream error: "
74 << QuicErrorCodeToString(error_code) << " " << error_message;
75 encoder_stream_error_detected_ = true;
76 }
77
ParseInputFilename(absl::string_view input_filename)78 bool QpackOfflineDecoder::ParseInputFilename(absl::string_view input_filename) {
79 std::vector<absl::string_view> pieces = absl::StrSplit(input_filename, '.');
80
81 if (pieces.size() < 3) {
82 QUIC_LOG(ERROR) << "Not enough fields in input filename " << input_filename;
83 return false;
84 }
85
86 auto piece_it = pieces.rbegin();
87
88 // Acknowledgement mode: 1 for immediate, 0 for none.
89 if (*piece_it != "0" && *piece_it != "1") {
90 QUIC_LOG(ERROR)
91 << "Header acknowledgement field must be 0 or 1 in input filename "
92 << input_filename;
93 return false;
94 }
95
96 ++piece_it;
97
98 // Maximum allowed number of blocked streams.
99 uint64_t max_blocked_streams = 0;
100 if (!absl::SimpleAtoi(*piece_it, &max_blocked_streams)) {
101 QUIC_LOG(ERROR) << "Error parsing part of input filename \"" << *piece_it
102 << "\" as an integer.";
103 return false;
104 }
105
106 ++piece_it;
107
108 // Maximum Dynamic Table Capacity in bytes
109 uint64_t maximum_dynamic_table_capacity = 0;
110 if (!absl::SimpleAtoi(*piece_it, &maximum_dynamic_table_capacity)) {
111 QUIC_LOG(ERROR) << "Error parsing part of input filename \"" << *piece_it
112 << "\" as an integer.";
113 return false;
114 }
115 qpack_decoder_ = std::make_unique<QpackDecoder>(
116 maximum_dynamic_table_capacity, max_blocked_streams, this);
117 qpack_decoder_->set_qpack_stream_sender_delegate(
118 &decoder_stream_sender_delegate_);
119
120 // The initial dynamic table capacity is zero according to
121 // https://quicwg.org/base-drafts/draft-ietf-quic-qpack.html#eviction.
122 // However, for historical reasons, offline interop encoders use
123 // |maximum_dynamic_table_capacity| as initial capacity.
124 qpack_decoder_->OnSetDynamicTableCapacity(maximum_dynamic_table_capacity);
125
126 return true;
127 }
128
DecodeHeaderBlocksFromFile(absl::string_view input_filename)129 bool QpackOfflineDecoder::DecodeHeaderBlocksFromFile(
130 absl::string_view input_filename) {
131 // Store data in |input_data_storage|; use a absl::string_view to
132 // efficiently keep track of remaining portion yet to be decoded.
133 std::optional<std::string> input_data_storage =
134 quiche::ReadFileContents(input_filename);
135 QUICHE_DCHECK(input_data_storage.has_value());
136 absl::string_view input_data(*input_data_storage);
137
138 while (!input_data.empty()) {
139 // Parse stream_id and length.
140 if (input_data.size() < sizeof(uint64_t) + sizeof(uint32_t)) {
141 QUIC_LOG(ERROR) << "Unexpected end of input file.";
142 return false;
143 }
144
145 uint64_t stream_id = quiche::QuicheEndian::NetToHost64(
146 *reinterpret_cast<const uint64_t*>(input_data.data()));
147 input_data = input_data.substr(sizeof(uint64_t));
148
149 uint32_t length = quiche::QuicheEndian::NetToHost32(
150 *reinterpret_cast<const uint32_t*>(input_data.data()));
151 input_data = input_data.substr(sizeof(uint32_t));
152
153 if (input_data.size() < length) {
154 QUIC_LOG(ERROR) << "Unexpected end of input file.";
155 return false;
156 }
157
158 // Parse data.
159 absl::string_view data = input_data.substr(0, length);
160 input_data = input_data.substr(length);
161
162 // Process data.
163 if (stream_id == 0) {
164 qpack_decoder_->encoder_stream_receiver()->Decode(data);
165
166 if (encoder_stream_error_detected_) {
167 QUIC_LOG(ERROR) << "Error detected on encoder stream.";
168 return false;
169 }
170 } else {
171 auto headers_handler = std::make_unique<test::TestHeadersHandler>();
172 auto progressive_decoder = qpack_decoder_->CreateProgressiveDecoder(
173 stream_id, headers_handler.get());
174
175 progressive_decoder->Decode(data);
176 progressive_decoder->EndHeaderBlock();
177
178 if (headers_handler->decoding_error_detected()) {
179 QUIC_LOG(ERROR) << "Sync decoding error on stream " << stream_id << ": "
180 << headers_handler->error_message();
181 return false;
182 }
183
184 decoders_.push_back({std::move(headers_handler),
185 std::move(progressive_decoder), stream_id});
186 }
187
188 // Move decoded header lists from TestHeadersHandlers and append them to
189 // |decoded_header_lists_| while preserving the order in |decoders_|.
190 while (!decoders_.empty() &&
191 decoders_.front().headers_handler->decoding_completed()) {
192 Decoder* decoder = &decoders_.front();
193
194 if (decoder->headers_handler->decoding_error_detected()) {
195 QUIC_LOG(ERROR) << "Async decoding error on stream "
196 << decoder->stream_id << ": "
197 << decoder->headers_handler->error_message();
198 return false;
199 }
200
201 if (!decoder->headers_handler->decoding_completed()) {
202 QUIC_LOG(ERROR) << "Decoding incomplete after reading entire"
203 " file, on stream "
204 << decoder->stream_id;
205 return false;
206 }
207
208 decoded_header_lists_.push_back(
209 decoder->headers_handler->ReleaseHeaderList());
210 decoders_.pop_front();
211 }
212 }
213
214 if (!decoders_.empty()) {
215 QUICHE_DCHECK(!decoders_.front().headers_handler->decoding_completed());
216
217 QUIC_LOG(ERROR) << "Blocked decoding uncomplete after reading entire"
218 " file, on stream "
219 << decoders_.front().stream_id;
220 return false;
221 }
222
223 return true;
224 }
225
VerifyDecodedHeaderLists(absl::string_view expected_headers_filename)226 bool QpackOfflineDecoder::VerifyDecodedHeaderLists(
227 absl::string_view expected_headers_filename) {
228 // Store data in |expected_headers_data_storage|; use a
229 // absl::string_view to efficiently keep track of remaining portion
230 // yet to be decoded.
231 std::optional<std::string> expected_headers_data_storage =
232 quiche::ReadFileContents(expected_headers_filename);
233 QUICHE_DCHECK(expected_headers_data_storage.has_value());
234 absl::string_view expected_headers_data(*expected_headers_data_storage);
235
236 while (!decoded_header_lists_.empty()) {
237 spdy::Http2HeaderBlock decoded_header_list =
238 std::move(decoded_header_lists_.front());
239 decoded_header_lists_.pop_front();
240
241 spdy::Http2HeaderBlock expected_header_list;
242 if (!ReadNextExpectedHeaderList(&expected_headers_data,
243 &expected_header_list)) {
244 QUIC_LOG(ERROR)
245 << "Error parsing expected header list to match next decoded "
246 "header list.";
247 return false;
248 }
249
250 if (!CompareHeaderBlocks(std::move(decoded_header_list),
251 std::move(expected_header_list))) {
252 QUIC_LOG(ERROR) << "Decoded header does not match expected header.";
253 return false;
254 }
255 }
256
257 if (!expected_headers_data.empty()) {
258 QUIC_LOG(ERROR)
259 << "Not enough encoded header lists to match expected ones.";
260 return false;
261 }
262
263 return true;
264 }
265
ReadNextExpectedHeaderList(absl::string_view * expected_headers_data,spdy::Http2HeaderBlock * expected_header_list)266 bool QpackOfflineDecoder::ReadNextExpectedHeaderList(
267 absl::string_view* expected_headers_data,
268 spdy::Http2HeaderBlock* expected_header_list) {
269 while (true) {
270 absl::string_view::size_type endline = expected_headers_data->find('\n');
271
272 // Even last header list must be followed by an empty line.
273 if (endline == absl::string_view::npos) {
274 QUIC_LOG(ERROR) << "Unexpected end of expected header list file.";
275 return false;
276 }
277
278 if (endline == 0) {
279 // Empty line indicates end of header list.
280 *expected_headers_data = expected_headers_data->substr(1);
281 return true;
282 }
283
284 absl::string_view header_field = expected_headers_data->substr(0, endline);
285 std::vector<absl::string_view> pieces = absl::StrSplit(header_field, '\t');
286
287 if (pieces.size() != 2) {
288 QUIC_LOG(ERROR) << "Header key and value must be separated by TAB.";
289 return false;
290 }
291
292 expected_header_list->AppendValueOrAddHeader(pieces[0], pieces[1]);
293
294 *expected_headers_data = expected_headers_data->substr(endline + 1);
295 }
296 }
297
CompareHeaderBlocks(spdy::Http2HeaderBlock decoded_header_list,spdy::Http2HeaderBlock expected_header_list)298 bool QpackOfflineDecoder::CompareHeaderBlocks(
299 spdy::Http2HeaderBlock decoded_header_list,
300 spdy::Http2HeaderBlock expected_header_list) {
301 if (decoded_header_list == expected_header_list) {
302 return true;
303 }
304
305 // The h2o decoder reshuffles the "content-length" header and pseudo-headers,
306 // see
307 // https://github.com/qpackers/qifs/blob/master/encoded/qpack-03/h2o/README.md.
308 // Remove such headers one by one if they match.
309 const char* kContentLength = "content-length";
310 const char* kPseudoHeaderPrefix = ":";
311 for (spdy::Http2HeaderBlock::iterator decoded_it =
312 decoded_header_list.begin();
313 decoded_it != decoded_header_list.end();) {
314 const absl::string_view key = decoded_it->first;
315 if (key != kContentLength && !absl::StartsWith(key, kPseudoHeaderPrefix)) {
316 ++decoded_it;
317 continue;
318 }
319 spdy::Http2HeaderBlock::iterator expected_it =
320 expected_header_list.find(key);
321 if (expected_it == expected_header_list.end() ||
322 decoded_it->second != expected_it->second) {
323 ++decoded_it;
324 continue;
325 }
326 // Http2HeaderBlock does not support erasing by iterator, only by key.
327 ++decoded_it;
328 expected_header_list.erase(key);
329 // This will invalidate |key|.
330 decoded_header_list.erase(key);
331 }
332
333 return decoded_header_list == expected_header_list;
334 }
335
336 } // namespace quic
337