1 // Copyright 2011 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/dns/serial_worker.h"
6
7 #include <memory>
8 #include <utility>
9
10 #include "base/check.h"
11 #include "base/functional/bind.h"
12 #include "base/functional/callback.h"
13 #include "base/location.h"
14 #include "base/memory/raw_ptr.h"
15 #include "base/run_loop.h"
16 #include "base/synchronization/lock.h"
17 #include "base/synchronization/waitable_event.h"
18 #include "base/task/current_thread.h"
19 #include "base/task/single_thread_task_runner.h"
20 #include "base/test/simple_test_tick_clock.h"
21 #include "base/threading/thread_restrictions.h"
22 #include "base/time/time.h"
23 #include "base/timer/timer.h"
24 #include "net/base/backoff_entry.h"
25 #include "net/test/test_with_task_environment.h"
26 #include "testing/gtest/include/gtest/gtest.h"
27
28 namespace net {
29
30 namespace {
31 constexpr base::TimeDelta kBackoffInitialDelay = base::Milliseconds(100);
32 constexpr int kBackoffMultiplyFactor = 2;
33 constexpr int kMaxRetries = 3;
34
35 static const BackoffEntry::Policy kTestBackoffPolicy = {
36 0, // Number of initial errors to ignore without backoff.
37 static_cast<int>(
38 kBackoffInitialDelay
39 .InMilliseconds()), // Initial delay for backoff in ms.
40 kBackoffMultiplyFactor, // Factor to multiply for exponential backoff.
41 0, // Fuzzing percentage.
42 static_cast<int>(
43 base::Seconds(1).InMilliseconds()), // Maximum time to delay requests
44 // in ms: 1 second.
45 -1, // Don't discard entry.
46 false // Don't use initial delay unless the last was an error.
47 };
48
49 class SerialWorkerTest : public TestWithTaskEnvironment {
50 public:
51 // The class under test
52 class TestSerialWorker : public SerialWorker {
53 public:
54 class TestWorkItem : public SerialWorker::WorkItem {
55 public:
TestWorkItem(SerialWorkerTest * test)56 explicit TestWorkItem(SerialWorkerTest* test) : test_(test) {}
57
DoWork()58 void DoWork() override {
59 ASSERT_TRUE(test_);
60 test_->OnWork();
61 }
62
FollowupWork(base::OnceClosure closure)63 void FollowupWork(base::OnceClosure closure) override {
64 ASSERT_TRUE(test_);
65 test_->OnFollowup(std::move(closure));
66 }
67
68 private:
69 raw_ptr<SerialWorkerTest> test_;
70 };
71
TestSerialWorker(SerialWorkerTest * t)72 explicit TestSerialWorker(SerialWorkerTest* t)
73 : SerialWorker(/*max_number_of_retries=*/kMaxRetries,
74 &kTestBackoffPolicy),
75 test_(t) {}
76 ~TestSerialWorker() override = default;
77
CreateWorkItem()78 std::unique_ptr<SerialWorker::WorkItem> CreateWorkItem() override {
79 return std::make_unique<TestWorkItem>(test_);
80 }
81
OnWorkFinished(std::unique_ptr<SerialWorker::WorkItem> work_item)82 bool OnWorkFinished(
83 std::unique_ptr<SerialWorker::WorkItem> work_item) override {
84 CHECK(test_);
85 return test_->OnWorkFinished();
86 }
87
88 private:
89 raw_ptr<SerialWorkerTest> test_;
90 };
91
92 SerialWorkerTest(const SerialWorkerTest&) = delete;
93 SerialWorkerTest& operator=(const SerialWorkerTest&) = delete;
94
95 // Mocks
96
OnWork()97 void OnWork() {
98 { // Check that OnWork is executed serially.
99 base::AutoLock lock(work_lock_);
100 EXPECT_FALSE(work_running_) << "`DoWork()` is not called serially!";
101 work_running_ = true;
102 }
103 num_work_calls_observed_++;
104 BreakNow("OnWork");
105 {
106 base::ScopedAllowBaseSyncPrimitivesForTesting
107 scoped_allow_base_sync_primitives;
108 work_allowed_.Wait();
109 }
110 // Calling from ThreadPool, but protected by work_allowed_/work_called_.
111 output_value_ = input_value_;
112
113 { // This lock might be destroyed after work_called_ is signalled.
114 base::AutoLock lock(work_lock_);
115 work_running_ = false;
116 }
117 work_called_.Signal();
118 }
119
OnFollowup(base::OnceClosure closure)120 void OnFollowup(base::OnceClosure closure) {
121 EXPECT_TRUE(task_runner_->BelongsToCurrentThread());
122
123 followup_closure_ = std::move(closure);
124 BreakNow("OnFollowup");
125
126 if (followup_immediately_)
127 CompleteFollowup();
128 }
129
OnWorkFinished()130 bool OnWorkFinished() {
131 EXPECT_TRUE(task_runner_->BelongsToCurrentThread());
132 EXPECT_EQ(output_value_, input_value_);
133 ++work_finished_calls_;
134 BreakNow("OnWorkFinished");
135 return on_work_finished_should_report_success_;
136 }
137
138 protected:
BreakCallback(const std::string & breakpoint)139 void BreakCallback(const std::string& breakpoint) {
140 breakpoint_ = breakpoint;
141 run_loop_->Quit();
142 }
143
BreakNow(const std::string & b)144 void BreakNow(const std::string& b) {
145 task_runner_->PostTask(FROM_HERE,
146 base::BindOnce(&SerialWorkerTest::BreakCallback,
147 base::Unretained(this), b));
148 }
149
RunUntilBreak(const std::string & b)150 void RunUntilBreak(const std::string& b) {
151 base::RunLoop run_loop;
152 ASSERT_FALSE(run_loop_);
153 run_loop_ = &run_loop;
154 run_loop_->Run();
155 run_loop_ = nullptr;
156 ASSERT_EQ(breakpoint_, b);
157 }
158
CompleteFollowup()159 void CompleteFollowup() {
160 ASSERT_TRUE(followup_closure_);
161 task_runner_->PostTask(FROM_HERE, std::move(followup_closure_));
162 }
163
SerialWorkerTest()164 SerialWorkerTest()
165 : TestWithTaskEnvironment(
166 base::test::TaskEnvironment::TimeSource::MOCK_TIME),
167 work_allowed_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
168 base::WaitableEvent::InitialState::NOT_SIGNALED),
169 work_called_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
170 base::WaitableEvent::InitialState::NOT_SIGNALED) {}
171
172 // Helpers for tests.
173
174 // Lets OnWork run and waits for it to complete. Can only return if OnWork is
175 // executed on a concurrent thread. Before calling, OnWork() must already have
176 // been started and blocked (ensured by running `RunUntilBreak("OnWork")`).
UnblockWork()177 void UnblockWork() {
178 ASSERT_TRUE(work_running_);
179 work_allowed_.Signal();
180 work_called_.Wait();
181 }
182
183 // test::Test methods
SetUp()184 void SetUp() override {
185 task_runner_ = base::SingleThreadTaskRunner::GetCurrentDefault();
186 }
187
TearDown()188 void TearDown() override {
189 // Cancel the worker to catch if it makes a late DoWork call.
190 if (worker_)
191 worker_->Cancel();
192 // Check if OnWork is stalled.
193 EXPECT_FALSE(work_running_) << "OnWork should be done by TearDown";
194 // Release it for cleanliness.
195 if (work_running_) {
196 UnblockWork();
197 }
198 }
199
200 // Input value read on WorkerPool.
201 int input_value_ = 0;
202 // Output value written on WorkerPool.
203 int output_value_ = -1;
204 // The number of times we saw an OnWork call.
205 int num_work_calls_observed_ = 0;
206 bool on_work_finished_should_report_success_ = true;
207
208 // read is called on WorkerPool so we need to synchronize with it.
209 base::WaitableEvent work_allowed_;
210 base::WaitableEvent work_called_;
211
212 // Protected by read_lock_. Used to verify that read calls are serialized.
213 bool work_running_ = false;
214 base::Lock work_lock_;
215
216 int work_finished_calls_ = 0;
217
218 // Task runner for this thread.
219 scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
220
221 // WatcherDelegate under test.
222 std::unique_ptr<TestSerialWorker> worker_ =
223 std::make_unique<TestSerialWorker>(this);
224
225 std::string breakpoint_;
226 raw_ptr<base::RunLoop> run_loop_ = nullptr;
227
228 bool followup_immediately_ = true;
229 base::OnceClosure followup_closure_;
230 };
231
TEST_F(SerialWorkerTest,RunWorkMultipleTimes)232 TEST_F(SerialWorkerTest, RunWorkMultipleTimes) {
233 for (int i = 0; i < 3; ++i) {
234 ++input_value_;
235 worker_->WorkNow();
236 RunUntilBreak("OnWork");
237 EXPECT_EQ(work_finished_calls_, i);
238 UnblockWork();
239 RunUntilBreak("OnFollowup");
240 RunUntilBreak("OnWorkFinished");
241 EXPECT_EQ(work_finished_calls_, i + 1);
242
243 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
244 }
245 }
246
TEST_F(SerialWorkerTest,TriggerTwoTimesBeforeRun)247 TEST_F(SerialWorkerTest, TriggerTwoTimesBeforeRun) {
248 // Schedule two calls. OnWork checks if it is called serially.
249 ++input_value_;
250 worker_->WorkNow();
251 // Work is blocked, so this will have to induce re-work
252 worker_->WorkNow();
253
254 // Expect 2 cycles through work.
255 RunUntilBreak("OnWork");
256 UnblockWork();
257 RunUntilBreak("OnWork");
258 UnblockWork();
259 RunUntilBreak("OnFollowup");
260 RunUntilBreak("OnWorkFinished");
261
262 EXPECT_EQ(work_finished_calls_, 1);
263
264 // No more tasks should remain.
265 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
266 }
267
TEST_F(SerialWorkerTest,TriggerThreeTimesBeforeRun)268 TEST_F(SerialWorkerTest, TriggerThreeTimesBeforeRun) {
269 // Schedule two calls. OnWork checks if it is called serially.
270 ++input_value_;
271 worker_->WorkNow();
272 // Work is blocked, so this will have to induce re-work
273 worker_->WorkNow();
274 // Repeat work is already scheduled, so this should be a noop.
275 worker_->WorkNow();
276
277 // Expect 2 cycles through work.
278 RunUntilBreak("OnWork");
279 UnblockWork();
280 RunUntilBreak("OnWork");
281 UnblockWork();
282 RunUntilBreak("OnFollowup");
283 RunUntilBreak("OnWorkFinished");
284
285 EXPECT_EQ(work_finished_calls_, 1);
286
287 // No more tasks should remain.
288 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
289 }
290
TEST_F(SerialWorkerTest,DelayFollowupCompletion)291 TEST_F(SerialWorkerTest, DelayFollowupCompletion) {
292 followup_immediately_ = false;
293 worker_->WorkNow();
294
295 RunUntilBreak("OnWork");
296 UnblockWork();
297 RunUntilBreak("OnFollowup");
298 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
299
300 CompleteFollowup();
301 RunUntilBreak("OnWorkFinished");
302
303 EXPECT_EQ(work_finished_calls_, 1);
304
305 // No more tasks should remain.
306 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
307 }
308
TEST_F(SerialWorkerTest,RetriggerDuringRun)309 TEST_F(SerialWorkerTest, RetriggerDuringRun) {
310 // Trigger work and wait until blocked.
311 worker_->WorkNow();
312 RunUntilBreak("OnWork");
313
314 worker_->WorkNow();
315 worker_->WorkNow();
316
317 // Expect a second work cycle after completion of current.
318 UnblockWork();
319 RunUntilBreak("OnWork");
320 UnblockWork();
321 RunUntilBreak("OnFollowup");
322 RunUntilBreak("OnWorkFinished");
323
324 EXPECT_EQ(work_finished_calls_, 1);
325
326 // No more tasks should remain.
327 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
328 }
329
TEST_F(SerialWorkerTest,RetriggerDuringFollowup)330 TEST_F(SerialWorkerTest, RetriggerDuringFollowup) {
331 // Trigger work and wait until blocked on followup.
332 followup_immediately_ = false;
333 worker_->WorkNow();
334 RunUntilBreak("OnWork");
335 UnblockWork();
336 RunUntilBreak("OnFollowup");
337
338 worker_->WorkNow();
339 worker_->WorkNow();
340
341 // Expect a second work cycle after completion of followup.
342 CompleteFollowup();
343 RunUntilBreak("OnWork");
344 UnblockWork();
345 RunUntilBreak("OnFollowup");
346 CompleteFollowup();
347 RunUntilBreak("OnWorkFinished");
348
349 EXPECT_EQ(work_finished_calls_, 1);
350
351 // No more tasks should remain.
352 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
353 }
354
TEST_F(SerialWorkerTest,CancelDuringWork)355 TEST_F(SerialWorkerTest, CancelDuringWork) {
356 worker_->WorkNow();
357
358 RunUntilBreak("OnWork");
359
360 worker_->Cancel();
361 UnblockWork();
362
363 RunUntilIdle();
364 EXPECT_EQ(breakpoint_, "OnWork");
365
366 EXPECT_EQ(work_finished_calls_, 0);
367
368 // No more tasks should remain.
369 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
370 }
371
TEST_F(SerialWorkerTest,CancelDuringFollowup)372 TEST_F(SerialWorkerTest, CancelDuringFollowup) {
373 followup_immediately_ = false;
374 worker_->WorkNow();
375
376 RunUntilBreak("OnWork");
377 UnblockWork();
378 RunUntilBreak("OnFollowup");
379
380 worker_->Cancel();
381 CompleteFollowup();
382
383 RunUntilIdle();
384 EXPECT_EQ(breakpoint_, "OnFollowup");
385
386 EXPECT_EQ(work_finished_calls_, 0);
387
388 // No more tasks should remain.
389 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
390 }
391
TEST_F(SerialWorkerTest,DeleteDuringWork)392 TEST_F(SerialWorkerTest, DeleteDuringWork) {
393 worker_->WorkNow();
394
395 RunUntilBreak("OnWork");
396
397 worker_.reset();
398 UnblockWork();
399
400 RunUntilIdle();
401 EXPECT_EQ(breakpoint_, "OnWork");
402
403 EXPECT_EQ(work_finished_calls_, 0);
404
405 // No more tasks should remain.
406 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
407 }
408
TEST_F(SerialWorkerTest,DeleteDuringFollowup)409 TEST_F(SerialWorkerTest, DeleteDuringFollowup) {
410 followup_immediately_ = false;
411 worker_->WorkNow();
412
413 RunUntilBreak("OnWork");
414 UnblockWork();
415 RunUntilBreak("OnFollowup");
416
417 worker_.reset();
418 CompleteFollowup();
419
420 RunUntilIdle();
421 EXPECT_EQ(breakpoint_, "OnFollowup");
422
423 EXPECT_EQ(work_finished_calls_, 0);
424
425 // No more tasks should remain.
426 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
427 }
428
TEST_F(SerialWorkerTest,RetryAndThenSucceed)429 TEST_F(SerialWorkerTest, RetryAndThenSucceed) {
430 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
431
432 // Induce a failure.
433 on_work_finished_should_report_success_ = false;
434 ++input_value_;
435 worker_->WorkNow();
436 RunUntilBreak("OnWork");
437 UnblockWork();
438 RunUntilBreak("OnFollowup");
439 RunUntilBreak("OnWorkFinished");
440
441 // Confirm it failed and that a retry was scheduled.
442 ASSERT_EQ(1, worker_->GetBackoffEntryForTesting().failure_count());
443 EXPECT_EQ(kBackoffInitialDelay,
444 worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
445
446 // Make the subsequent attempt succeed.
447 on_work_finished_should_report_success_ = true;
448
449 RunUntilBreak("OnWork");
450 UnblockWork();
451 RunUntilBreak("OnFollowup");
452 RunUntilBreak("OnWorkFinished");
453 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
454
455 EXPECT_EQ(2, num_work_calls_observed_);
456
457 // No more tasks should remain.
458 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
459 }
460
TEST_F(SerialWorkerTest,ExternalWorkRequestResetsRetryState)461 TEST_F(SerialWorkerTest, ExternalWorkRequestResetsRetryState) {
462 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
463
464 // Induce a failure.
465 on_work_finished_should_report_success_ = false;
466 ++input_value_;
467 worker_->WorkNow();
468 RunUntilBreak("OnWork");
469 UnblockWork();
470 RunUntilBreak("OnFollowup");
471 RunUntilBreak("OnWorkFinished");
472
473 // Confirm it failed and that a retry was scheduled.
474 ASSERT_EQ(1, worker_->GetBackoffEntryForTesting().failure_count());
475 EXPECT_TRUE(worker_->GetRetryTimerForTesting().IsRunning());
476 EXPECT_EQ(kBackoffInitialDelay,
477 worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
478 on_work_finished_should_report_success_ = true;
479
480 // The retry state should be reset before we see OnWorkFinished.
481 worker_->WorkNow();
482 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
483 EXPECT_FALSE(worker_->GetRetryTimerForTesting().IsRunning());
484 EXPECT_EQ(base::TimeDelta(),
485 worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
486 RunUntilBreak("OnWork");
487 UnblockWork();
488 RunUntilBreak("OnFollowup");
489 RunUntilBreak("OnWorkFinished");
490
491 // No more tasks should remain.
492 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
493 }
494
TEST_F(SerialWorkerTest,MultipleFailureExponentialBackoff)495 TEST_F(SerialWorkerTest, MultipleFailureExponentialBackoff) {
496 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
497
498 // Induce a failure.
499 on_work_finished_should_report_success_ = false;
500 ++input_value_;
501 worker_->WorkNow();
502 RunUntilBreak("OnWork");
503 UnblockWork();
504 RunUntilBreak("OnFollowup");
505 RunUntilBreak("OnWorkFinished");
506
507 for (int retry_attempt_count = 1; retry_attempt_count <= kMaxRetries;
508 retry_attempt_count++) {
509 // Confirm it failed and that a retry was scheduled.
510 ASSERT_EQ(retry_attempt_count,
511 worker_->GetBackoffEntryForTesting().failure_count());
512 EXPECT_TRUE(worker_->GetRetryTimerForTesting().IsRunning());
513 base::TimeDelta expected_backoff_delay;
514 if (retry_attempt_count == 1) {
515 expected_backoff_delay = kBackoffInitialDelay;
516 } else {
517 expected_backoff_delay = kBackoffInitialDelay * kBackoffMultiplyFactor *
518 (retry_attempt_count - 1);
519 }
520 EXPECT_EQ(expected_backoff_delay,
521 worker_->GetBackoffEntryForTesting().GetTimeUntilRelease())
522 << "retry_attempt_count=" << retry_attempt_count;
523
524 // |on_work_finished_should_report_success_| is still false, so the retry
525 // will fail too
526 RunUntilBreak("OnWork");
527 UnblockWork();
528 RunUntilBreak("OnFollowup");
529 RunUntilBreak("OnWorkFinished");
530 }
531
532 // The last retry attempt resets the retry state.
533 ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
534 EXPECT_FALSE(worker_->GetRetryTimerForTesting().IsRunning());
535 EXPECT_EQ(base::TimeDelta(),
536 worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
537 on_work_finished_should_report_success_ = true;
538
539 // No more tasks should remain.
540 EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
541 }
542
543 } // namespace
544
545 } // namespace net
546