1 : // Copyright 2014 Google Inc. All Rights Reserved.
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : #include "syzygy/kasko/report_repository.h"
16 :
17 : #include <algorithm>
18 : #include <map>
19 : #include <string>
20 : #include <utility>
21 : #include <vector>
22 : #include "base/bind.h"
23 : #include "base/macros.h"
24 : #include "base/rand_util.h"
25 : #include "base/files/file_enumerator.h"
26 : #include "base/files/file_path.h"
27 : #include "base/files/file_util.h"
28 : #include "base/files/scoped_temp_dir.h"
29 : #include "base/strings/string16.h"
30 : #include "base/strings/string_number_conversions.h"
31 : #include "base/strings/utf_string_conversions.h"
32 : #include "base/time/time.h"
33 : #include "gtest/gtest.h"
34 : #include "syzygy/kasko/crash_keys_serialization.h"
35 :
36 : namespace kasko {
37 :
38 : namespace {
39 :
40 : // This test harness allows us to generate reports and mock the results of
41 : // upload attempts for them. A report may be configured to succeed immediately,
42 : // succeed after 1 or 2 retries, or fail permanently.
43 : // The test harness will log failures if the permanent failure or upload
44 : // handler is invoked inappropriately or not invoked when expected.
45 : // A mock TimeSource is used to simulate the passage of time for retry
46 : // intervals.
47 : // Each test should call repository()->UploadPendingReport() enough times to
48 : // empty the repository. The harness expects it to be empty at the end of the
49 : // test.
50 : class ReportRepositoryTest : public testing::Test {
51 : public:
52 : static const uint16_t kHalfRetryIntervalInSeconds;
53 : static const uint16_t kRetryIntervalInSeconds;
54 :
55 : // The mock time must not start at 0, as we cannot update a file timestamp to
56 : // that value.
57 E : ReportRepositoryTest()
58 : : remainder_expected_(false), time_(base::Time::Now()) {}
59 :
60 : protected:
61 : typedef std::pair<std::string, std::map<base::string16, base::string16>>
62 : Report;
63 :
64 : // testing::Test implementation
65 E : void SetUp() override {
66 E : repository_temp_dir_.CreateUniqueTempDir();
67 : repository_.reset(new ReportRepository(
68 : repository_temp_dir_.path(),
69 : base::TimeDelta::FromSeconds(kRetryIntervalInSeconds),
70 : base::Bind(&ReportRepositoryTest::GetTime, base::Unretained(this)),
71 : base::Bind(&ReportRepositoryTest::Upload, base::Unretained(this)),
72 : base::Bind(&ReportRepositoryTest::HandlePermanentFailure,
73 E : base::Unretained(this))));
74 E : }
75 E : void TearDown() override { Validate(); }
76 :
77 : // Validates that all injected reports have been handled as expected, and that
78 : // the repository directory does not contain any leftover files.
79 : // This is automatically called by TearDown but may also be invoked mid-test.
80 E : void Validate() {
81 : // There should not be anything left over.
82 E : EXPECT_EQ(base::FilePath(),
83 : base::FileEnumerator(repository_temp_dir_.path(), true,
84 : base::FileEnumerator::FILES).Next());
85 :
86 : // If |remainder_expected_| is true, we allow exactly one report to not be
87 : // processed (due to corruption).
88 E : for (size_t i = 0; i < arraysize(successful_reports_); ++i) {
89 E : if (remainder_expected_ && successful_reports_[i].size() == 1) {
90 E : remainder_expected_ = false;
91 E : successful_reports_[i].clear();
92 E : continue;
93 : }
94 E : EXPECT_TRUE(successful_reports_[i].empty());
95 E : }
96 E : for (size_t i = 0; i < arraysize(failing_reports_); ++i) {
97 E : if (remainder_expected_ && failing_reports_[i].size() == 1) {
98 i : remainder_expected_ = false;
99 i : failing_reports_[i].clear();
100 i : continue;
101 : }
102 E : EXPECT_TRUE(failing_reports_[i].empty());
103 E : }
104 :
105 : // |remainder_expected_| should have been reset during the above loops.
106 E : EXPECT_FALSE(remainder_expected_);
107 E : }
108 :
109 : // Indicates that one report has been intentionally corrupted. This will be
110 : // checked during Validate().
111 E : void SetRemainderExpected() { remainder_expected_ = true; }
112 :
113 : // Randomly deletes a report file (either crash keys or minidump) from the
114 : // repository.
115 E : void OrphanAReport() {
116 E : base::FilePath to_delete;
117 E : size_t count = 0;
118 : base::FileEnumerator file_enumerator(repository_temp_dir_.path(), true,
119 E : base::FileEnumerator::FILES);
120 E : for (base::FilePath candidate = file_enumerator.Next(); !candidate.empty();
121 E : candidate = file_enumerator.Next()) {
122 E : ++count;
123 E : if (base::RandDouble() < 1.0 / count)
124 E : to_delete = candidate;
125 E : }
126 E : ASSERT_FALSE(to_delete.empty());
127 E : ASSERT_TRUE(base::DeleteFile(to_delete, false));
128 E : }
129 :
130 : // Implements the TimeSource.
131 E : base::Time GetTime() { return time_; }
132 :
133 : // Increments the simulated clock.
134 E : void IncrementTime(const base::TimeDelta& time_delta) { time_ += time_delta; }
135 :
136 : // Creates a report that will succeed after the specified number of retries
137 : // (0, 1, or 2).
138 E : void InjectForSuccessAfterRetries(size_t retries) {
139 E : Report report = GenerateReport();
140 E : AllowReportToSucceedAfterRetries(report, retries);
141 E : StoreReport(report);
142 E : }
143 :
144 : // Creates a report that will never succeed in uploading.
145 E : void InjectForFailure() {
146 E : Report report = GenerateReport();
147 E : PermanentlyFailReport(report);
148 E : StoreReport(report);
149 E : }
150 :
151 : // Returns the instance under test.
152 E : ReportRepository* repository() { return repository_.get(); }
153 :
154 : private:
155 : // Writes a report to disk and stores it in the repository.
156 E : void StoreReport(const Report& report) {
157 E : base::FilePath minidump_file;
158 E : ASSERT_TRUE(base::CreateTemporaryFileInDir(repository_temp_dir_.path(),
159 : &minidump_file));
160 E : ASSERT_TRUE(base::WriteFile(minidump_file, report.first.data(),
161 : report.first.length()));
162 E : repository_->StoreReport(minidump_file, report.second);
163 E : }
164 :
165 : // Sets up the mock behaviour for a report that will succeed after the
166 : // specified number of retries.
167 E : void AllowReportToSucceedAfterRetries(const Report& report, size_t retries) {
168 E : ASSERT_LT(retries, arraysize(successful_reports_));
169 E : successful_reports_[retries].push_back(report);
170 E : }
171 :
172 : // Sets up the mock behaviour for a report that will always fail to upload.
173 E : void PermanentlyFailReport(const Report& report) {
174 E : failing_reports_[3].push_back(report);
175 E : }
176 :
177 : // Generates a unique report.
178 E : Report GenerateReport() {
179 : static size_t id = 0;
180 E : Report report;
181 E : report.first = base::UintToString(id);
182 E : report.second[L"id"] = base::ASCIIToUTF16(report.first);
183 E : ++id;
184 E : return report;
185 E : }
186 :
187 : // Implements the UploadHandler.
188 : bool Upload(const base::FilePath& minidump_path,
189 E : const std::map<base::string16, base::string16>& crash_keys) {
190 E : Report report;
191 E : bool success = base::ReadFileToString(minidump_path, &report.first);
192 E : EXPECT_TRUE(success);
193 E : if (!success)
194 i : return false;
195 :
196 E : report.second = crash_keys;
197 :
198 : // Check to see if this report is destined to eventually succeed. If it's in
199 : // successful_reports_[0] it succeeds this round. If it's in [1] or higher
200 : // it will fail this round but be advanced to a lower index to eventually
201 : // succeed.
202 E : for (size_t i = 0; i < arraysize(successful_reports_); ++i) {
203 : std::vector<Report>::iterator entry = std::find(
204 E : successful_reports_[i].begin(), successful_reports_[i].end(), report);
205 E : if (entry == successful_reports_[i].end())
206 E : continue;
207 :
208 : // Remove it from whence it was found.
209 E : successful_reports_[i].erase(entry);
210 : // Advance it, if necessary.
211 E : if (i > 0) {
212 E : successful_reports_[i-1].push_back(report);
213 E : return false;
214 : }
215 E : return true;
216 i : }
217 :
218 : // Check to see if this report is destined for permanent failure.
219 : // Start at [1] because the elements in [0] are ready for
220 : // HandlePermanentFailure.
221 E : for (size_t i = 1; i < arraysize(failing_reports_); ++i) {
222 : std::vector<Report>::iterator entry = std::find(
223 E : failing_reports_[i].begin(), failing_reports_[i].end(), report);
224 E : if (entry == failing_reports_[i].end())
225 E : continue;
226 : // Remove it from whence it was found.
227 E : failing_reports_[i].erase(entry);
228 : // Advance towards later permanent failure.
229 E : failing_reports_[i - 1].push_back(report);
230 E : return false;
231 i : }
232 i : ADD_FAILURE() << "Unexpected report. Minidump: " << report.first;
233 i : return false;
234 E : }
235 :
236 : // Implements the PermanentFailureHandler.
237 : void HandlePermanentFailure(const base::FilePath& minidump_path,
238 E : const base::FilePath& crash_keys_path) {
239 E : Report report;
240 E : EXPECT_TRUE(ReadCrashKeysFromFile(crash_keys_path, &report.second));
241 E : ASSERT_TRUE(base::ReadFileToString(minidump_path, &report.first));
242 :
243 : std::vector<Report>::iterator entry = std::find(
244 E : failing_reports_[0].begin(), failing_reports_[0].end(), report);
245 E : if (entry == failing_reports_[0].end()) {
246 i : ADD_FAILURE() << "Unexpected permanently failed report. Minidump: "
247 : << report.first;
248 i : } else {
249 E : failing_reports_[0].erase(entry);
250 : }
251 E : }
252 :
253 : // If true, exactly one report should never have been sent (because we
254 : // corrupted it).
255 : bool remainder_expected_;
256 :
257 : // Vectors of reports that should succeed after 0, 1, or 2 failures according
258 : // to their index in this array.
259 : std::vector<Report> successful_reports_[3];
260 :
261 : // Vectors of reports that should permanently fail after 1, 2, or 3 more
262 : // failures according to their index in this array. Index [0] is reports that
263 : // have just failed upload and should now be handed to the
264 : // PermanentFailureHandler.
265 : std::vector<Report> failing_reports_[4];
266 :
267 : // The repository directory.
268 : base::ScopedTempDir repository_temp_dir_;
269 :
270 : // The mock time.
271 : base::Time time_;
272 :
273 : // The instance under test.
274 : scoped_ptr<ReportRepository> repository_;
275 :
276 : DISALLOW_COPY_AND_ASSIGN(ReportRepositoryTest);
277 : };
278 :
279 : const uint16_t ReportRepositoryTest::kHalfRetryIntervalInSeconds = 10;
280 : const uint16_t ReportRepositoryTest::kRetryIntervalInSeconds =
281 : ReportRepositoryTest::kHalfRetryIntervalInSeconds * 2;
282 :
283 : } // namespace
284 :
285 E : TEST_F(ReportRepositoryTest, BasicTest) {
286 E : EXPECT_FALSE(repository()->HasPendingReports());
287 :
288 E : InjectForSuccessAfterRetries(2);
289 E : EXPECT_TRUE(repository()->HasPendingReports());
290 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
291 E : EXPECT_FALSE(repository()->HasPendingReports());
292 :
293 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
294 :
295 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
296 E : EXPECT_TRUE(repository()->HasPendingReports());
297 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
298 E : EXPECT_FALSE(repository()->HasPendingReports());
299 :
300 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
301 E : EXPECT_TRUE(repository()->HasPendingReports());
302 E : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
303 E : EXPECT_FALSE(repository()->HasPendingReports());
304 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
305 :
306 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
307 E : EXPECT_FALSE(repository()->HasPendingReports());
308 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
309 E : }
310 :
311 E : TEST_F(ReportRepositoryTest, SuccessTest) {
312 E : EXPECT_FALSE(repository()->HasPendingReports());
313 :
314 E : InjectForSuccessAfterRetries(0);
315 E : EXPECT_TRUE(repository()->HasPendingReports());
316 E : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
317 E : EXPECT_FALSE(repository()->HasPendingReports());
318 :
319 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
320 :
321 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
322 E : EXPECT_FALSE(repository()->HasPendingReports());
323 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
324 E : }
325 :
326 E : TEST_F(ReportRepositoryTest, PermanentFailureTest) {
327 E : EXPECT_FALSE(repository()->HasPendingReports());
328 :
329 E : InjectForFailure();
330 E : EXPECT_TRUE(repository()->HasPendingReports());
331 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
332 E : EXPECT_FALSE(repository()->HasPendingReports());
333 :
334 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
335 E : EXPECT_TRUE(repository()->HasPendingReports());
336 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
337 E : EXPECT_FALSE(repository()->HasPendingReports());
338 :
339 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
340 E : EXPECT_TRUE(repository()->HasPendingReports());
341 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
342 E : EXPECT_FALSE(repository()->HasPendingReports());
343 E : }
344 :
345 E : TEST_F(ReportRepositoryTest, MultipleReportsTest) {
346 E : EXPECT_FALSE(repository()->HasPendingReports());
347 :
348 E : InjectForSuccessAfterRetries(0);
349 E : InjectForSuccessAfterRetries(0);
350 E : InjectForSuccessAfterRetries(0);
351 :
352 E : EXPECT_TRUE(repository()->HasPendingReports());
353 E : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
354 E : EXPECT_TRUE(repository()->HasPendingReports());
355 E : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
356 E : EXPECT_TRUE(repository()->HasPendingReports());
357 E : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
358 E : EXPECT_FALSE(repository()->HasPendingReports());
359 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
360 :
361 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
362 E : EXPECT_FALSE(repository()->HasPendingReports());
363 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
364 E : }
365 :
366 E : TEST_F(ReportRepositoryTest, MultipleReportsTestWithFailures) {
367 E : EXPECT_FALSE(repository()->HasPendingReports());
368 :
369 E : InjectForSuccessAfterRetries(0);
370 E : InjectForSuccessAfterRetries(1);
371 E : InjectForSuccessAfterRetries(2);
372 E : InjectForFailure();
373 :
374 : // 3 will fail, 1 will succeed.
375 E : size_t successes = 0;
376 E : for (size_t i = 0; i < 4; ++i) {
377 E : EXPECT_TRUE(repository()->HasPendingReports());
378 E : if (repository()->UploadPendingReport())
379 E : successes++;
380 E : }
381 E : EXPECT_EQ(1, successes);
382 E : EXPECT_FALSE(repository()->HasPendingReports());
383 E : EXPECT_TRUE(repository()->UploadPendingReport());
384 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
385 :
386 : // 2 will fail, 1 will succeed.
387 E : successes = 0;
388 E : for (size_t i = 0; i < 3; ++i) {
389 E : EXPECT_TRUE(repository()->HasPendingReports());
390 E : if (repository()->UploadPendingReport())
391 E : successes++;
392 E : }
393 E : EXPECT_EQ(1, successes);
394 E : EXPECT_FALSE(repository()->HasPendingReports());
395 E : EXPECT_TRUE(repository()->UploadPendingReport());
396 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
397 :
398 : // 1 will permanently fail, 1 will succeed.
399 E : successes = 0;
400 E : for (size_t i = 0; i < 2; ++i) {
401 E : EXPECT_TRUE(repository()->HasPendingReports());
402 E : if (repository()->UploadPendingReport())
403 E : successes++;
404 E : }
405 E : EXPECT_EQ(1, successes);
406 E : EXPECT_FALSE(repository()->HasPendingReports());
407 E : EXPECT_TRUE(repository()->UploadPendingReport());
408 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
409 :
410 : // None left.
411 E : EXPECT_FALSE(repository()->HasPendingReports());
412 E : EXPECT_TRUE(repository()->UploadPendingReport());
413 E : }
414 :
415 E : TEST_F(ReportRepositoryTest, MultipleInterleavedReports) {
416 E : EXPECT_FALSE(repository()->HasPendingReports());
417 :
418 : // 1st generation
419 E : InjectForSuccessAfterRetries(1);
420 E : InjectForSuccessAfterRetries(2);
421 :
422 E : EXPECT_TRUE(repository()->HasPendingReports());
423 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
424 E : EXPECT_TRUE(repository()->HasPendingReports());
425 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
426 E : EXPECT_FALSE(repository()->HasPendingReports());
427 :
428 : // Increment a half interval.
429 E : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
430 E : EXPECT_FALSE(repository()->HasPendingReports());
431 : // 2nd generation
432 E : InjectForSuccessAfterRetries(1);
433 E : InjectForSuccessAfterRetries(2);
434 E : EXPECT_TRUE(repository()->HasPendingReports());
435 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
436 E : EXPECT_TRUE(repository()->HasPendingReports());
437 E : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
438 E : EXPECT_FALSE(repository()->HasPendingReports());
439 E : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
440 :
441 : // Increment another half interval. Now only the first generation are eligible
442 : // for retry. One will succeed.
443 E : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
444 E : size_t successes = 0;
445 E : for (size_t i = 0; i < 2; ++i) {
446 E : EXPECT_TRUE(repository()->HasPendingReports());
447 E : if (repository()->UploadPendingReport())
448 E : successes++;
449 E : }
450 E : EXPECT_EQ(1, successes);
451 E : EXPECT_FALSE(repository()->HasPendingReports());
452 :
453 : // Increment another half interval. This is the second generation, one will
454 : // succeed.
455 E : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
456 E : successes = 0;
457 E : for (size_t i = 0; i < 2; ++i) {
458 E : EXPECT_TRUE(repository()->HasPendingReports());
459 E : if (repository()->UploadPendingReport())
460 E : successes++;
461 E : }
462 E : EXPECT_EQ(1, successes);
463 E : EXPECT_FALSE(repository()->HasPendingReports());
464 :
465 : // Increment another half interval. This is the first generation, only one
466 : // element left (it will succeed).
467 E : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
468 E : EXPECT_TRUE(repository()->HasPendingReports());
469 E : EXPECT_TRUE(repository()->UploadPendingReport());
470 E : EXPECT_FALSE(repository()->HasPendingReports());
471 :
472 : // Increment another half interval. This is the second generation, only one
473 : // element left (it will succeed).
474 E : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
475 E : EXPECT_TRUE(repository()->HasPendingReports());
476 E : EXPECT_TRUE(repository()->UploadPendingReport());
477 E : EXPECT_FALSE(repository()->HasPendingReports());
478 E : }
479 :
480 E : TEST_F(ReportRepositoryTest, CorruptionTest) {
481 : // In order to avoid hard-coding extensions/paths, and having a bunch of
482 : // permutations, let's run this test a bunch of times and probabilistically
483 : // cover all the cases of a file being missing.
484 E : for (size_t i = 0; i< 100; ++i) {
485 : // This sequence will put one report each in the different states.
486 E : InjectForSuccessAfterRetries(2); // one in Incoming
487 E : InjectForSuccessAfterRetries(2); // two in Incoming
488 E : repository()->UploadPendingReport(); // one in Retry
489 E : repository()->UploadPendingReport(); // two in Retry
490 E : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
491 E : repository()->UploadPendingReport(); // one in Retry 2
492 E : InjectForSuccessAfterRetries(2); // one in Incoming
493 :
494 : // Randomly delete one file.
495 E : OrphanAReport();
496 :
497 : // Wait 36 hours.
498 E : base::Time start = GetTime();
499 E : while (GetTime() - start < base::TimeDelta::FromHours(36)) {
500 E : IncrementTime(base::TimeDelta::FromMinutes(30));
501 E : repository()->UploadPendingReport();
502 E : }
503 :
504 E : SetRemainderExpected();
505 : // Validate that exactly one of the injected reports didn't come out and
506 : // that there are no files left over.
507 E : Validate();
508 E : }
509 E : }
510 :
511 : } // namespace kasko
|