1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/cloud/curl_http_request.h"
17
18 #include <fstream>
19 #include <string>
20
21 #include "tensorflow/core/lib/core/status_test_util.h"
22 #include "tensorflow/core/platform/mem.h"
23 #include "tensorflow/core/platform/path.h"
24 #include "tensorflow/core/platform/test.h"
25
26 namespace tensorflow {
27 namespace {
28
29 const string kTestContent = "random original scratch content";
30
31 class FakeEnv : public EnvWrapper {
32 public:
FakeEnv()33 FakeEnv() : EnvWrapper(Env::Default()) {}
34
NowSeconds() const35 uint64 NowSeconds() const override { return now_; }
36 uint64 now_ = 10000;
37 };
38
39 // A fake proxy that pretends to be libcurl.
40 class FakeLibCurl : public LibCurl {
41 public:
FakeLibCurl(const string & response_content,uint64 response_code)42 FakeLibCurl(const string& response_content, uint64 response_code)
43 : response_content_(response_content), response_code_(response_code) {}
FakeLibCurl(const string & response_content,uint64 response_code,std::vector<std::tuple<uint64,curl_off_t>> progress_ticks,FakeEnv * env)44 FakeLibCurl(const string& response_content, uint64 response_code,
45 std::vector<std::tuple<uint64, curl_off_t>> progress_ticks,
46 FakeEnv* env)
47 : response_content_(response_content),
48 response_code_(response_code),
49 progress_ticks_(std::move(progress_ticks)),
50 env_(env) {}
FakeLibCurl(const string & response_content,uint64 response_code,const std::vector<string> & response_headers)51 FakeLibCurl(const string& response_content, uint64 response_code,
52 const std::vector<string>& response_headers)
53 : response_content_(response_content),
54 response_code_(response_code),
55 response_headers_(response_headers) {}
curl_easy_init()56 CURL* curl_easy_init() override {
57 is_initialized_ = true;
58 // The reuslt just needs to be non-null.
59 return reinterpret_cast<CURL*>(this);
60 }
curl_easy_setopt(CURL * curl,CURLoption option,uint64 param)61 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
62 uint64 param) override {
63 switch (option) {
64 case CURLOPT_POST:
65 is_post_ = param;
66 break;
67 case CURLOPT_PUT:
68 is_put_ = param;
69 break;
70 default:
71 break;
72 }
73 return CURLE_OK;
74 }
curl_easy_setopt(CURL * curl,CURLoption option,const char * param)75 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
76 const char* param) override {
77 return curl_easy_setopt(curl, option,
78 reinterpret_cast<void*>(const_cast<char*>(param)));
79 }
curl_easy_setopt(CURL * curl,CURLoption option,void * param)80 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
81 void* param) override {
82 switch (option) {
83 case CURLOPT_URL:
84 url_ = reinterpret_cast<char*>(param);
85 break;
86 case CURLOPT_RANGE:
87 range_ = reinterpret_cast<char*>(param);
88 break;
89 case CURLOPT_CUSTOMREQUEST:
90 custom_request_ = reinterpret_cast<char*>(param);
91 break;
92 case CURLOPT_HTTPHEADER:
93 headers_ = reinterpret_cast<std::vector<string>*>(param);
94 break;
95 case CURLOPT_ERRORBUFFER:
96 error_buffer_ = reinterpret_cast<char*>(param);
97 break;
98 case CURLOPT_CAINFO:
99 ca_info_ = reinterpret_cast<char*>(param);
100 break;
101 case CURLOPT_WRITEDATA:
102 write_data_ = reinterpret_cast<FILE*>(param);
103 break;
104 case CURLOPT_HEADERDATA:
105 header_data_ = reinterpret_cast<FILE*>(param);
106 break;
107 case CURLOPT_READDATA:
108 read_data_ = reinterpret_cast<FILE*>(param);
109 break;
110 case CURLOPT_XFERINFODATA:
111 progress_data_ = param;
112 break;
113 default:
114 break;
115 }
116 return CURLE_OK;
117 }
curl_easy_setopt(CURL * curl,CURLoption option,size_t (* param)(void *,size_t,size_t,FILE *))118 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
119 size_t (*param)(void*, size_t, size_t,
120 FILE*)) override {
121 read_callback_ = param;
122 return CURLE_OK;
123 }
curl_easy_setopt(CURL * curl,CURLoption option,size_t (* param)(const void *,size_t,size_t,void *))124 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
125 size_t (*param)(const void*, size_t, size_t,
126 void*)) override {
127 switch (option) {
128 case CURLOPT_WRITEFUNCTION:
129 write_callback_ = param;
130 break;
131 case CURLOPT_HEADERFUNCTION:
132 header_callback_ = param;
133 break;
134 default:
135 break;
136 }
137 return CURLE_OK;
138 }
curl_easy_setopt(CURL * curl,CURLoption option,int (* param)(void * clientp,curl_off_t dltotal,curl_off_t dlnow,curl_off_t ultotal,curl_off_t ulnow))139 CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
140 int (*param)(void* clientp, curl_off_t dltotal,
141 curl_off_t dlnow, curl_off_t ultotal,
142 curl_off_t ulnow)) override {
143 progress_callback_ = param;
144 return CURLE_OK;
145 }
curl_easy_perform(CURL * curl)146 CURLcode curl_easy_perform(CURL* curl) override {
147 if (is_post_ || is_put_) {
148 char buffer[3];
149 int bytes_read;
150 posted_content_ = "";
151 do {
152 bytes_read = read_callback_(buffer, 1, sizeof(buffer), read_data_);
153 posted_content_ =
154 strings::StrCat(posted_content_, StringPiece(buffer, bytes_read));
155 } while (bytes_read > 0);
156 }
157 if (write_data_ || write_callback_) {
158 size_t bytes_handled = write_callback_(
159 response_content_.c_str(), 1, response_content_.size(), write_data_);
160 // Mimic real libcurl behavior by checking write callback return value.
161 if (bytes_handled != response_content_.size()) {
162 curl_easy_perform_result_ = CURLE_WRITE_ERROR;
163 }
164 }
165 for (const auto& header : response_headers_) {
166 header_callback_(header.c_str(), 1, header.size(), header_data_);
167 }
168 if (error_buffer_) {
169 strncpy(error_buffer_, curl_easy_perform_error_message_.c_str(),
170 curl_easy_perform_error_message_.size() + 1);
171 }
172 for (const auto& tick : progress_ticks_) {
173 env_->now_ = std::get<0>(tick);
174 if (progress_callback_(progress_data_, 0, std::get<1>(tick), 0, 0)) {
175 return CURLE_ABORTED_BY_CALLBACK;
176 }
177 }
178 return curl_easy_perform_result_;
179 }
curl_easy_getinfo(CURL * curl,CURLINFO info,uint64 * value)180 CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
181 uint64* value) override {
182 switch (info) {
183 case CURLINFO_RESPONSE_CODE:
184 *value = response_code_;
185 break;
186 default:
187 break;
188 }
189 return CURLE_OK;
190 }
curl_easy_getinfo(CURL * curl,CURLINFO info,double * value)191 CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
192 double* value) override {
193 switch (info) {
194 case CURLINFO_SIZE_DOWNLOAD:
195 *value = response_content_.size();
196 break;
197 default:
198 break;
199 }
200 return CURLE_OK;
201 }
curl_easy_cleanup(CURL * curl)202 void curl_easy_cleanup(CURL* curl) override { is_cleaned_up_ = true; }
curl_slist_append(curl_slist * list,const char * str)203 curl_slist* curl_slist_append(curl_slist* list, const char* str) override {
204 std::vector<string>* v = list ? reinterpret_cast<std::vector<string>*>(list)
205 : new std::vector<string>();
206 v->push_back(str);
207 return reinterpret_cast<curl_slist*>(v);
208 }
curl_easy_escape(CURL * curl,const char * str,int length)209 char* curl_easy_escape(CURL* curl, const char* str, int length) override {
210 // This function just does a simple replacing of "/" with "%2F" instead of
211 // full url encoding.
212 const string victim = "/";
213 const string encoded = "%2F";
214
215 string temp_str = str;
216 std::string::size_type n = 0;
217 while ((n = temp_str.find(victim, n)) != std::string::npos) {
218 temp_str.replace(n, victim.size(), encoded);
219 n += encoded.size();
220 }
221 char* out_char_str = reinterpret_cast<char*>(
222 port::Malloc(sizeof(char) * temp_str.size() + 1));
223 std::copy(temp_str.begin(), temp_str.end(), out_char_str);
224 out_char_str[temp_str.size()] = '\0';
225 return out_char_str;
226 }
curl_slist_free_all(curl_slist * list)227 void curl_slist_free_all(curl_slist* list) override {
228 delete reinterpret_cast<std::vector<string>*>(list);
229 }
curl_free(void * p)230 void curl_free(void* p) override { port::Free(p); }
231
232 // Variables defining the behavior of this fake.
233 string response_content_;
234 uint64 response_code_;
235 std::vector<string> response_headers_;
236
237 // Internal variables to store the libcurl state.
238 string url_;
239 string range_;
240 string custom_request_;
241 string ca_info_;
242 char* error_buffer_ = nullptr;
243 bool is_initialized_ = false;
244 bool is_cleaned_up_ = false;
245 std::vector<string>* headers_ = nullptr;
246 bool is_post_ = false;
247 bool is_put_ = false;
248 void* write_data_ = nullptr;
249 size_t (*write_callback_)(const void* ptr, size_t size, size_t nmemb,
250 void* userdata) = nullptr;
251 void* header_data_ = nullptr;
252 size_t (*header_callback_)(const void* ptr, size_t size, size_t nmemb,
253 void* userdata) = nullptr;
254 FILE* read_data_ = nullptr;
255 size_t (*read_callback_)(void* ptr, size_t size, size_t nmemb,
256 FILE* userdata) = &fread;
257 int (*progress_callback_)(void* clientp, curl_off_t dltotal, curl_off_t dlnow,
258 curl_off_t ultotal, curl_off_t ulnow) = nullptr;
259 void* progress_data_ = nullptr;
260 // Outcome of performing the request.
261 string posted_content_;
262 CURLcode curl_easy_perform_result_ = CURLE_OK;
263 string curl_easy_perform_error_message_;
264 // A vector of <timestamp, progress in bytes> pairs that represent the
265 // progress of a transmission.
266 std::vector<std::tuple<uint64, curl_off_t>> progress_ticks_;
267 FakeEnv* env_ = nullptr;
268 };
269
TEST(CurlHttpRequestTest,GetRequest)270 TEST(CurlHttpRequestTest, GetRequest) {
271 FakeLibCurl libcurl("get response", 200);
272 CurlHttpRequest http_request(&libcurl);
273
274 std::vector<char> scratch;
275 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
276 scratch.reserve(100);
277
278 http_request.SetUri("http://www.testuri.com");
279 http_request.AddAuthBearerHeader("fake-bearer");
280 http_request.SetRange(100, 199);
281 http_request.SetResultBuffer(&scratch);
282 TF_EXPECT_OK(http_request.Send());
283
284 EXPECT_EQ("get response", string(scratch.begin(), scratch.end()));
285
286 // Check interactions with libcurl.
287 EXPECT_TRUE(libcurl.is_initialized_);
288 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
289 EXPECT_EQ("100-199", libcurl.range_);
290 EXPECT_EQ("", libcurl.custom_request_);
291 EXPECT_EQ("", libcurl.ca_info_);
292 EXPECT_EQ(1, libcurl.headers_->size());
293 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
294 EXPECT_FALSE(libcurl.is_post_);
295 EXPECT_EQ(200, http_request.GetResponseCode());
296 }
297
TEST(CurlHttpRequestTest,GetRequest_Direct)298 TEST(CurlHttpRequestTest, GetRequest_Direct) {
299 FakeLibCurl libcurl("get response", 200);
300 CurlHttpRequest http_request(&libcurl);
301
302 std::vector<char> scratch(100, 0);
303
304 http_request.SetUri("http://www.testuri.com");
305 http_request.AddAuthBearerHeader("fake-bearer");
306 http_request.SetRange(100, 199);
307 http_request.SetResultBufferDirect(scratch.data(), scratch.capacity());
308 TF_EXPECT_OK(http_request.Send());
309
310 string expected_response = "get response";
311 size_t response_bytes_transferred =
312 http_request.GetResultBufferDirectBytesTransferred();
313 EXPECT_EQ(expected_response.size(), response_bytes_transferred);
314 EXPECT_EQ(
315 "get response",
316 string(scratch.begin(), scratch.begin() + response_bytes_transferred));
317
318 // Check interactions with libcurl.
319 EXPECT_TRUE(libcurl.is_initialized_);
320 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
321 EXPECT_EQ("100-199", libcurl.range_);
322 EXPECT_EQ("", libcurl.custom_request_);
323 EXPECT_EQ("", libcurl.ca_info_);
324 EXPECT_EQ(1, libcurl.headers_->size());
325 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
326 EXPECT_FALSE(libcurl.is_post_);
327 EXPECT_EQ(200, http_request.GetResponseCode());
328 }
329
TEST(CurlHttpRequestTest,GetRequest_CustomCaInfoFlag)330 TEST(CurlHttpRequestTest, GetRequest_CustomCaInfoFlag) {
331 static char set_var[] = "CURL_CA_BUNDLE=test";
332 putenv(set_var);
333 FakeLibCurl libcurl("get response", 200);
334 CurlHttpRequest http_request(&libcurl);
335
336 std::vector<char> scratch;
337 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
338 scratch.reserve(100);
339
340 http_request.SetUri("http://www.testuri.com");
341 http_request.AddAuthBearerHeader("fake-bearer");
342 http_request.SetRange(100, 199);
343 http_request.SetResultBuffer(&scratch);
344 TF_EXPECT_OK(http_request.Send());
345
346 EXPECT_EQ("get response", string(scratch.begin(), scratch.end()));
347
348 // Check interactions with libcurl.
349 EXPECT_TRUE(libcurl.is_initialized_);
350 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
351 EXPECT_EQ("100-199", libcurl.range_);
352 EXPECT_EQ("", libcurl.custom_request_);
353 EXPECT_EQ("test", libcurl.ca_info_);
354 EXPECT_EQ(1, libcurl.headers_->size());
355 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
356 EXPECT_FALSE(libcurl.is_post_);
357 EXPECT_EQ(200, http_request.GetResponseCode());
358 }
359
TEST(CurlHttpRequestTest,GetRequest_Direct_ResponseTooLarge)360 TEST(CurlHttpRequestTest, GetRequest_Direct_ResponseTooLarge) {
361 FakeLibCurl libcurl("get response", 200);
362 CurlHttpRequest http_request(&libcurl);
363
364 std::vector<char> scratch(5, 0);
365
366 http_request.SetUri("http://www.testuri.com");
367 http_request.SetResultBufferDirect(scratch.data(), scratch.size());
368 const Status& status = http_request.Send();
369 EXPECT_EQ(error::FAILED_PRECONDITION, status.code());
370 EXPECT_EQ(
371 "Error executing an HTTP request: libcurl code 23 meaning "
372 "'Failed writing received data to disk/application', error details: "
373 "Received 12 response bytes for a 5-byte buffer",
374 status.error_message());
375
376 // As long as the request clearly fails, ok to leave truncated response here.
377 EXPECT_EQ(5, http_request.GetResultBufferDirectBytesTransferred());
378 EXPECT_EQ("get r", string(scratch.begin(), scratch.begin() + 5));
379 }
380
TEST(CurlHttpRequestTest,GetRequest_Direct_RangeOutOfBound)381 TEST(CurlHttpRequestTest, GetRequest_Direct_RangeOutOfBound) {
382 FakeLibCurl libcurl("get response", 416);
383 CurlHttpRequest http_request(&libcurl);
384
385 const string initialScratch = "abcde";
386 std::vector<char> scratch;
387 scratch.insert(scratch.end(), initialScratch.begin(), initialScratch.end());
388
389 http_request.SetUri("http://www.testuri.com");
390 http_request.SetRange(0, 4);
391 http_request.SetResultBufferDirect(scratch.data(), scratch.size());
392 TF_EXPECT_OK(http_request.Send());
393 EXPECT_EQ(416, http_request.GetResponseCode());
394
395 // Some servers (in particular, GCS) return an error message payload with a
396 // 416 Range Not Satisfiable response. We should pretend it's not there when
397 // reporting bytes transferred, but it's ok if it writes to scratch.
398 EXPECT_EQ(0, http_request.GetResultBufferDirectBytesTransferred());
399 EXPECT_EQ("get r", string(scratch.begin(), scratch.end()));
400 }
401
TEST(CurlHttpRequestTest,GetRequest_Empty)402 TEST(CurlHttpRequestTest, GetRequest_Empty) {
403 FakeLibCurl libcurl("", 200);
404 CurlHttpRequest http_request(&libcurl);
405
406 std::vector<char> scratch;
407 scratch.resize(0);
408
409 http_request.SetUri("http://www.testuri.com");
410 http_request.AddAuthBearerHeader("fake-bearer");
411 http_request.SetRange(100, 199);
412 http_request.SetResultBuffer(&scratch);
413 TF_EXPECT_OK(http_request.Send());
414
415 EXPECT_TRUE(scratch.empty());
416
417 // Check interactions with libcurl.
418 EXPECT_TRUE(libcurl.is_initialized_);
419 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
420 EXPECT_EQ("100-199", libcurl.range_);
421 EXPECT_EQ("", libcurl.custom_request_);
422 EXPECT_EQ(1, libcurl.headers_->size());
423 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
424 EXPECT_FALSE(libcurl.is_post_);
425 EXPECT_EQ(200, http_request.GetResponseCode());
426 }
427
TEST(CurlHttpRequestTest,GetRequest_RangeOutOfBound)428 TEST(CurlHttpRequestTest, GetRequest_RangeOutOfBound) {
429 FakeLibCurl libcurl("get response", 416);
430 CurlHttpRequest http_request(&libcurl);
431
432 std::vector<char> scratch;
433 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
434
435 http_request.SetUri("http://www.testuri.com");
436 http_request.AddAuthBearerHeader("fake-bearer");
437 http_request.SetRange(100, 199);
438 http_request.SetResultBuffer(&scratch);
439 TF_EXPECT_OK(http_request.Send());
440
441 // Some servers (in particular, GCS) return an error message payload with a
442 // 416 Range Not Satisfiable response. We should pretend it's not there.
443 EXPECT_TRUE(scratch.empty());
444 EXPECT_EQ(416, http_request.GetResponseCode());
445 }
446
TEST(CurlHttpRequestTest,GetRequest_503)447 TEST(CurlHttpRequestTest, GetRequest_503) {
448 FakeLibCurl libcurl("get response", 503);
449 CurlHttpRequest http_request(&libcurl);
450
451 std::vector<char> scratch;
452 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
453
454 http_request.SetUri("http://www.testuri.com");
455 http_request.SetResultBuffer(&scratch);
456 const auto& status = http_request.Send();
457 EXPECT_EQ(error::UNAVAILABLE, status.code());
458 EXPECT_EQ(
459 "Error executing an HTTP request: HTTP response code 503 with body "
460 "'get response'",
461 status.error_message());
462 }
463
TEST(CurlHttpRequestTest,GetRequest_HttpCode0)464 TEST(CurlHttpRequestTest, GetRequest_HttpCode0) {
465 FakeLibCurl libcurl("get response", 0);
466 libcurl.curl_easy_perform_result_ = CURLE_OPERATION_TIMEDOUT;
467 libcurl.curl_easy_perform_error_message_ = "Operation timed out";
468 CurlHttpRequest http_request(&libcurl);
469
470 std::vector<char> scratch;
471 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
472
473 http_request.SetUri("http://www.testuri.com");
474 const auto& status = http_request.Send();
475 EXPECT_EQ(error::UNAVAILABLE, status.code());
476 EXPECT_EQ(
477 "Error executing an HTTP request: libcurl code 28 meaning "
478 "'Timeout was reached', error details: Operation timed out",
479 status.error_message());
480 EXPECT_EQ(0, http_request.GetResponseCode());
481 }
482
TEST(CurlHttpRequestTest,GetRequest_CouldntResolveHost)483 TEST(CurlHttpRequestTest, GetRequest_CouldntResolveHost) {
484 FakeLibCurl libcurl("get response", 0);
485 libcurl.curl_easy_perform_result_ = CURLE_COULDNT_RESOLVE_HOST;
486 libcurl.curl_easy_perform_error_message_ =
487 "Could not resolve host 'metadata'";
488 CurlHttpRequest http_request(&libcurl);
489
490 std::vector<char> scratch;
491 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
492
493 http_request.SetUri("http://metadata");
494 const auto& status = http_request.Send();
495 EXPECT_EQ(error::FAILED_PRECONDITION, status.code());
496 EXPECT_EQ(
497 "Error executing an HTTP request: libcurl code 6 meaning "
498 "'Couldn't resolve host name', error details: Could not resolve host "
499 "'metadata'",
500 status.error_message());
501 EXPECT_EQ(0, http_request.GetResponseCode());
502 }
503
TEST(CurlHttpRequestTest,GetRequest_SslBadCertfile)504 TEST(CurlHttpRequestTest, GetRequest_SslBadCertfile) {
505 FakeLibCurl libcurl("get response", 0);
506 libcurl.curl_easy_perform_result_ = CURLE_SSL_CACERT_BADFILE;
507 libcurl.curl_easy_perform_error_message_ =
508 "error setting certificate verify locations:";
509 CurlHttpRequest http_request(&libcurl);
510
511 std::vector<char> scratch;
512 scratch.insert(scratch.end(), kTestContent.begin(), kTestContent.end());
513
514 http_request.SetUri("http://metadata");
515 const auto& status = http_request.Send();
516 EXPECT_EQ(error::FAILED_PRECONDITION, status.code());
517 EXPECT_EQ(
518 "Error executing an HTTP request: libcurl code 77 meaning "
519 "'Problem with the SSL CA cert (path? access rights?)', error details: "
520 "error setting certificate verify locations:",
521 status.error_message());
522 EXPECT_EQ(0, http_request.GetResponseCode());
523 }
524
TEST(CurlHttpRequestTest,ResponseHeaders)525 TEST(CurlHttpRequestTest, ResponseHeaders) {
526 FakeLibCurl libcurl(
527 "get response", 200,
528 {"Location: abcd", "Content-Type: text", "unparsable header"});
529 CurlHttpRequest http_request(&libcurl);
530
531 http_request.SetUri("http://www.testuri.com");
532 TF_EXPECT_OK(http_request.Send());
533
534 EXPECT_EQ("abcd", http_request.GetResponseHeader("Location"));
535 EXPECT_EQ("text", http_request.GetResponseHeader("Content-Type"));
536 EXPECT_EQ("", http_request.GetResponseHeader("Not-Seen-Header"));
537 }
538
TEST(CurlHttpRequestTest,PutRequest_WithBody_FromFile)539 TEST(CurlHttpRequestTest, PutRequest_WithBody_FromFile) {
540 FakeLibCurl libcurl("", 200);
541 CurlHttpRequest http_request(&libcurl);
542
543 auto content_filename = io::JoinPath(testing::TmpDir(), "content");
544 std::ofstream content(content_filename, std::ofstream::binary);
545 content << "post body content";
546 content.close();
547
548 http_request.SetUri("http://www.testuri.com");
549 http_request.AddAuthBearerHeader("fake-bearer");
550 TF_EXPECT_OK(http_request.SetPutFromFile(content_filename, 0));
551 TF_EXPECT_OK(http_request.Send());
552
553 // Check interactions with libcurl.
554 EXPECT_TRUE(libcurl.is_initialized_);
555 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
556 EXPECT_EQ("", libcurl.custom_request_);
557 EXPECT_EQ(2, libcurl.headers_->size());
558 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
559 EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]);
560 EXPECT_TRUE(libcurl.is_put_);
561 EXPECT_EQ("post body content", libcurl.posted_content_);
562
563 std::remove(content_filename.c_str());
564 }
565
TEST(CurlHttpRequestTest,PutRequest_WithBody_FromFile_NonZeroOffset)566 TEST(CurlHttpRequestTest, PutRequest_WithBody_FromFile_NonZeroOffset) {
567 FakeLibCurl libcurl("", 200);
568 CurlHttpRequest http_request(&libcurl);
569
570 auto content_filename = io::JoinPath(testing::TmpDir(), "content");
571 std::ofstream content(content_filename, std::ofstream::binary);
572 content << "post body content";
573 content.close();
574
575 http_request.SetUri("http://www.testuri.com");
576 http_request.AddAuthBearerHeader("fake-bearer");
577 TF_EXPECT_OK(http_request.SetPutFromFile(content_filename, 7));
578 TF_EXPECT_OK(http_request.Send());
579
580 // Check interactions with libcurl.
581 EXPECT_EQ("dy content", libcurl.posted_content_);
582
583 std::remove(content_filename.c_str());
584 }
585
TEST(CurlHttpRequestTest,PutRequest_WithoutBody)586 TEST(CurlHttpRequestTest, PutRequest_WithoutBody) {
587 FakeLibCurl libcurl("", 200);
588 CurlHttpRequest http_request(&libcurl);
589
590 http_request.SetUri("http://www.testuri.com");
591 http_request.AddAuthBearerHeader("fake-bearer");
592 http_request.SetPutEmptyBody();
593 TF_EXPECT_OK(http_request.Send());
594
595 // Check interactions with libcurl.
596 EXPECT_TRUE(libcurl.is_initialized_);
597 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
598 EXPECT_EQ("", libcurl.custom_request_);
599 EXPECT_EQ(3, libcurl.headers_->size());
600 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
601 EXPECT_EQ("Content-Length: 0", (*libcurl.headers_)[1]);
602 EXPECT_EQ("Transfer-Encoding: identity", (*libcurl.headers_)[2]);
603 EXPECT_TRUE(libcurl.is_put_);
604 EXPECT_EQ("", libcurl.posted_content_);
605 }
606
TEST(CurlHttpRequestTest,PostRequest_WithBody_FromMemory)607 TEST(CurlHttpRequestTest, PostRequest_WithBody_FromMemory) {
608 FakeLibCurl libcurl("", 200);
609 CurlHttpRequest http_request(&libcurl);
610
611 string content = "post body content";
612
613 http_request.SetUri("http://www.testuri.com");
614 http_request.AddAuthBearerHeader("fake-bearer");
615 http_request.SetPostFromBuffer(content.c_str(), content.size());
616 TF_EXPECT_OK(http_request.Send());
617
618 // Check interactions with libcurl.
619 EXPECT_TRUE(libcurl.is_initialized_);
620 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
621 EXPECT_EQ("", libcurl.custom_request_);
622 EXPECT_EQ(2, libcurl.headers_->size());
623 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
624 EXPECT_EQ("Content-Length: 17", (*libcurl.headers_)[1]);
625 EXPECT_TRUE(libcurl.is_post_);
626 EXPECT_EQ("post body content", libcurl.posted_content_);
627 }
628
TEST(CurlHttpRequestTest,PostRequest_WithoutBody)629 TEST(CurlHttpRequestTest, PostRequest_WithoutBody) {
630 FakeLibCurl libcurl("", 200);
631 CurlHttpRequest http_request(&libcurl);
632 http_request.SetUri("http://www.testuri.com");
633 http_request.AddAuthBearerHeader("fake-bearer");
634 http_request.SetPostEmptyBody();
635 TF_EXPECT_OK(http_request.Send());
636
637 // Check interactions with libcurl.
638 EXPECT_TRUE(libcurl.is_initialized_);
639 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
640 EXPECT_EQ("", libcurl.custom_request_);
641 EXPECT_EQ(3, libcurl.headers_->size());
642 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
643 EXPECT_EQ("Content-Length: 0", (*libcurl.headers_)[1]);
644 EXPECT_EQ("Transfer-Encoding: identity", (*libcurl.headers_)[2]);
645 EXPECT_TRUE(libcurl.is_post_);
646 EXPECT_EQ("", libcurl.posted_content_);
647 }
648
TEST(CurlHttpRequestTest,DeleteRequest)649 TEST(CurlHttpRequestTest, DeleteRequest) {
650 FakeLibCurl libcurl("", 200);
651 CurlHttpRequest http_request(&libcurl);
652 http_request.SetUri("http://www.testuri.com");
653 http_request.AddAuthBearerHeader("fake-bearer");
654 http_request.SetDeleteRequest();
655 TF_EXPECT_OK(http_request.Send());
656
657 // Check interactions with libcurl.
658 EXPECT_TRUE(libcurl.is_initialized_);
659 EXPECT_EQ("http://www.testuri.com", libcurl.url_);
660 EXPECT_EQ("DELETE", libcurl.custom_request_);
661 EXPECT_EQ(1, libcurl.headers_->size());
662 EXPECT_EQ("Authorization: Bearer fake-bearer", (*libcurl.headers_)[0]);
663 EXPECT_FALSE(libcurl.is_post_);
664 }
665
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_NoUri)666 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_NoUri) {
667 FakeLibCurl libcurl("", 200);
668 CurlHttpRequest http_request(&libcurl);
669 ASSERT_DEATH((void)http_request.Send(), "URI has not been set");
670 }
671
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_TwoSends)672 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_TwoSends) {
673 FakeLibCurl libcurl("", 200);
674 CurlHttpRequest http_request(&libcurl);
675 http_request.SetUri("http://www.google.com");
676 TF_EXPECT_OK(http_request.Send());
677 ASSERT_DEATH((void)http_request.Send(), "The request has already been sent");
678 }
679
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_ReusingAfterSend)680 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_ReusingAfterSend) {
681 FakeLibCurl libcurl("", 200);
682 CurlHttpRequest http_request(&libcurl);
683 http_request.SetUri("http://www.google.com");
684 TF_EXPECT_OK(http_request.Send());
685 ASSERT_DEATH(http_request.SetUri("http://mail.google.com"),
686 "The request has already been sent");
687 }
688
TEST(CurlHttpRequestTest,WrongSequenceOfCalls_SettingMethodTwice)689 TEST(CurlHttpRequestTest, WrongSequenceOfCalls_SettingMethodTwice) {
690 FakeLibCurl libcurl("", 200);
691 CurlHttpRequest http_request(&libcurl);
692 http_request.SetDeleteRequest();
693 ASSERT_DEATH(http_request.SetPostEmptyBody(),
694 "HTTP method has been already set");
695 }
696
TEST(CurlHttpRequestTest,EscapeString)697 TEST(CurlHttpRequestTest, EscapeString) {
698 FakeLibCurl libcurl("get response", 200);
699 CurlHttpRequest http_request(&libcurl);
700 const string test_string = "a/b/c";
701 EXPECT_EQ("a%2Fb%2Fc", http_request.EscapeString(test_string));
702 }
703
TEST(CurlHttpRequestTest,ErrorReturnsNoResponse)704 TEST(CurlHttpRequestTest, ErrorReturnsNoResponse) {
705 FakeLibCurl libcurl("get response", 500);
706 CurlHttpRequest http_request(&libcurl);
707
708 std::vector<char> scratch;
709 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
710 scratch.reserve(100);
711
712 http_request.SetUri("http://www.testuri.com");
713 http_request.AddAuthBearerHeader("fake-bearer");
714 http_request.SetRange(100, 199);
715 http_request.SetResultBuffer(&scratch);
716 EXPECT_EQ(error::UNAVAILABLE, http_request.Send().code());
717
718 EXPECT_EQ("", string(scratch.begin(), scratch.end()));
719 }
720
TEST(CurlHttpRequestTest,ProgressIsOk)721 TEST(CurlHttpRequestTest, ProgressIsOk) {
722 // Imitate a steady progress.
723 FakeEnv env;
724 FakeLibCurl libcurl(
725 "test", 200,
726 {
727 std::make_tuple(100, 0) /* timestamp 100, 0 bytes */,
728 std::make_tuple(110, 0) /* timestamp 110, 0 bytes */,
729 std::make_tuple(200, 100) /* timestamp 200, 100 bytes */
730 },
731 &env);
732 CurlHttpRequest http_request(&libcurl, &env);
733 http_request.SetUri("http://www.testuri.com");
734 TF_EXPECT_OK(http_request.Send());
735 }
736
TEST(CurlHttpRequestTest,ProgressIsStuck)737 TEST(CurlHttpRequestTest, ProgressIsStuck) {
738 // Imitate a transmission that got stuck for more than a minute.
739 FakeEnv env;
740 FakeLibCurl libcurl(
741 "test", 200,
742 {
743 std::make_tuple(100, 10) /* timestamp 100, 10 bytes */,
744 std::make_tuple(130, 10) /* timestamp 130, 10 bytes */,
745 std::make_tuple(170, 10) /* timestamp 170, 10 bytes */
746 },
747 &env);
748 CurlHttpRequest http_request(&libcurl, &env);
749 http_request.SetUri("http://www.testuri.com");
750 auto status = http_request.Send();
751 EXPECT_EQ(error::UNAVAILABLE, status.code());
752 EXPECT_EQ(
753 "Error executing an HTTP request: libcurl code 42 meaning 'Operation "
754 "was aborted by an application callback', error details: (none)",
755 status.error_message());
756 }
757
758 class TestStats : public HttpRequest::RequestStats {
759 public:
760 ~TestStats() override = default;
761
RecordRequest(const HttpRequest * request,const string & uri,HttpRequest::RequestMethod method)762 void RecordRequest(const HttpRequest* request, const string& uri,
763 HttpRequest::RequestMethod method) override {
764 has_recorded_request_ = true;
765 record_request_request_ = request;
766 record_request_uri_ = uri;
767 record_request_method_ = method;
768 }
769
RecordResponse(const HttpRequest * request,const string & uri,HttpRequest::RequestMethod method,const Status & result)770 void RecordResponse(const HttpRequest* request, const string& uri,
771 HttpRequest::RequestMethod method,
772 const Status& result) override {
773 has_recorded_response_ = true;
774 record_response_request_ = request;
775 record_response_uri_ = uri;
776 record_response_method_ = method;
777 record_response_result_ = result;
778 }
779
780 const HttpRequest* record_request_request_ = nullptr;
781 string record_request_uri_ = "http://www.testuri.com";
782 HttpRequest::RequestMethod record_request_method_ =
783 HttpRequest::RequestMethod::kGet;
784
785 const HttpRequest* record_response_request_ = nullptr;
786 string record_response_uri_ = "http://www.testuri.com";
787 HttpRequest::RequestMethod record_response_method_ =
788 HttpRequest::RequestMethod::kGet;
789 Status record_response_result_;
790
791 bool has_recorded_request_ = false;
792 bool has_recorded_response_ = false;
793 };
794
795 class StatsTestFakeLibCurl : public FakeLibCurl {
796 public:
StatsTestFakeLibCurl(TestStats * stats,const string & response_content,uint64 response_code)797 StatsTestFakeLibCurl(TestStats* stats, const string& response_content,
798 uint64 response_code)
799 : FakeLibCurl(response_content, response_code), stats_(stats) {}
curl_easy_perform(CURL * curl)800 CURLcode curl_easy_perform(CURL* curl) override {
801 CHECK(!performed_request_);
802 performed_request_ = true;
803 stats_had_recorded_request_ = stats_->has_recorded_request_;
804 stats_had_recorded_response_ = stats_->has_recorded_response_;
805 return FakeLibCurl::curl_easy_perform(curl);
806 };
807
808 TestStats* stats_;
809 bool performed_request_ = false;
810 bool stats_had_recorded_request_;
811 bool stats_had_recorded_response_;
812 };
813
TEST(CurlHttpRequestTest,StatsGetSuccessful)814 TEST(CurlHttpRequestTest, StatsGetSuccessful) {
815 TestStats stats;
816 StatsTestFakeLibCurl libcurl(&stats, "get response", 200);
817 CurlHttpRequest http_request(&libcurl);
818
819 std::vector<char> scratch;
820 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
821 scratch.reserve(100);
822
823 http_request.SetRequestStats(&stats);
824
825 http_request.SetUri("http://www.testuri.com");
826 http_request.AddAuthBearerHeader("fake-bearer");
827 http_request.SetRange(100, 199);
828 http_request.SetResultBuffer(&scratch);
829 TF_EXPECT_OK(http_request.Send());
830
831 EXPECT_EQ("get response", string(scratch.begin(), scratch.end()));
832
833 // Check interaction with stats.
834 ASSERT_TRUE(stats.has_recorded_request_);
835 EXPECT_EQ(&http_request, stats.record_request_request_);
836 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
837 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_request_method_);
838
839 ASSERT_TRUE(stats.has_recorded_response_);
840 EXPECT_EQ(&http_request, stats.record_response_request_);
841 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
842 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_response_method_);
843 TF_EXPECT_OK(stats.record_response_result_);
844
845 // Check interaction with libcurl.
846 EXPECT_TRUE(libcurl.performed_request_);
847 EXPECT_TRUE(libcurl.stats_had_recorded_request_);
848 EXPECT_FALSE(libcurl.stats_had_recorded_response_);
849 }
850
TEST(CurlHttpRequestTest,StatsGetNotFound)851 TEST(CurlHttpRequestTest, StatsGetNotFound) {
852 TestStats stats;
853 StatsTestFakeLibCurl libcurl(&stats, "get other response", 404);
854 CurlHttpRequest http_request(&libcurl);
855
856 std::vector<char> scratch;
857 scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end());
858 scratch.reserve(100);
859
860 http_request.SetRequestStats(&stats);
861
862 http_request.SetUri("http://www.testuri.com");
863 http_request.AddAuthBearerHeader("fake-bearer");
864 http_request.SetRange(100, 199);
865 http_request.SetResultBuffer(&scratch);
866 Status s = http_request.Send();
867
868 // Check interaction with stats.
869 ASSERT_TRUE(stats.has_recorded_request_);
870 EXPECT_EQ(&http_request, stats.record_request_request_);
871 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
872 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_request_method_);
873
874 ASSERT_TRUE(stats.has_recorded_response_);
875 EXPECT_EQ(&http_request, stats.record_response_request_);
876 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
877 EXPECT_EQ(HttpRequest::RequestMethod::kGet, stats.record_response_method_);
878 EXPECT_TRUE(errors::IsNotFound(stats.record_response_result_));
879 EXPECT_EQ(s, stats.record_response_result_);
880
881 // Check interaction with libcurl.
882 EXPECT_TRUE(libcurl.performed_request_);
883 EXPECT_TRUE(libcurl.stats_had_recorded_request_);
884 EXPECT_FALSE(libcurl.stats_had_recorded_response_);
885 }
886
TEST(CurlHttpRequestTest,StatsPost)887 TEST(CurlHttpRequestTest, StatsPost) {
888 TestStats stats;
889
890 FakeLibCurl libcurl("", 200);
891 CurlHttpRequest http_request(&libcurl);
892
893 http_request.SetRequestStats(&stats);
894
895 string content = "post body content";
896
897 http_request.SetUri("http://www.testuri.com");
898 http_request.SetPostFromBuffer(content.c_str(), content.size());
899 TF_EXPECT_OK(http_request.Send());
900
901 // Check interaction with stats.
902 ASSERT_TRUE(stats.has_recorded_request_);
903 EXPECT_EQ(&http_request, stats.record_request_request_);
904 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
905 EXPECT_EQ(HttpRequest::RequestMethod::kPost, stats.record_request_method_);
906
907 ASSERT_TRUE(stats.has_recorded_response_);
908 EXPECT_EQ(&http_request, stats.record_response_request_);
909 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
910 EXPECT_EQ(HttpRequest::RequestMethod::kPost, stats.record_response_method_);
911 TF_EXPECT_OK(stats.record_response_result_);
912 }
913
TEST(CurlHttpRequestTest,StatsDelete)914 TEST(CurlHttpRequestTest, StatsDelete) {
915 TestStats stats;
916
917 FakeLibCurl libcurl("", 200);
918 CurlHttpRequest http_request(&libcurl);
919 http_request.SetRequestStats(&stats);
920 http_request.SetUri("http://www.testuri.com");
921 http_request.SetDeleteRequest();
922 TF_EXPECT_OK(http_request.Send());
923
924 // Check interaction with stats.
925 ASSERT_TRUE(stats.has_recorded_request_);
926 EXPECT_EQ(&http_request, stats.record_request_request_);
927 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
928 EXPECT_EQ(HttpRequest::RequestMethod::kDelete, stats.record_request_method_);
929
930 ASSERT_TRUE(stats.has_recorded_response_);
931 EXPECT_EQ(&http_request, stats.record_response_request_);
932 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
933 EXPECT_EQ(HttpRequest::RequestMethod::kDelete, stats.record_response_method_);
934 TF_EXPECT_OK(stats.record_response_result_);
935 }
936
TEST(CurlHttpRequestTest,StatsPut)937 TEST(CurlHttpRequestTest, StatsPut) {
938 TestStats stats;
939
940 FakeLibCurl libcurl("", 200);
941 CurlHttpRequest http_request(&libcurl);
942 http_request.SetRequestStats(&stats);
943 http_request.SetUri("http://www.testuri.com");
944 http_request.AddAuthBearerHeader("fake-bearer");
945 http_request.SetPutEmptyBody();
946 TF_EXPECT_OK(http_request.Send());
947
948 // Check interaction with stats.
949 ASSERT_TRUE(stats.has_recorded_request_);
950 EXPECT_EQ(&http_request, stats.record_request_request_);
951 EXPECT_EQ("http://www.testuri.com", stats.record_request_uri_);
952 EXPECT_EQ(HttpRequest::RequestMethod::kPut, stats.record_request_method_);
953
954 ASSERT_TRUE(stats.has_recorded_response_);
955 EXPECT_EQ(&http_request, stats.record_response_request_);
956 EXPECT_EQ("http://www.testuri.com", stats.record_response_uri_);
957 EXPECT_EQ(HttpRequest::RequestMethod::kPut, stats.record_response_method_);
958 TF_EXPECT_OK(stats.record_response_result_);
959 }
960
961 } // namespace
962 } // namespace tensorflow
963