1 : // Copyright 2014 Google Inc. All Rights Reserved.
2 : //
3 : // Licensed under the Apache License, Version 2.0 (the "License");
4 : // you may not use this file except in compliance with the License.
5 : // You may obtain a copy of the License at
6 : //
7 : // http://www.apache.org/licenses/LICENSE-2.0
8 : //
9 : // Unless required by applicable law or agreed to in writing, software
10 : // distributed under the License is distributed on an "AS IS" BASIS,
11 : // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 : // See the License for the specific language governing permissions and
13 : // limitations under the License.
14 :
15 : #include "syzygy/kasko/report_repository.h"
16 :
17 : #include <algorithm>
18 : #include <map>
19 : #include <string>
20 : #include <utility>
21 : #include <vector>
22 : #include "base/bind.h"
23 : #include "base/macros.h"
24 : #include "base/rand_util.h"
25 : #include "base/files/file_enumerator.h"
26 : #include "base/files/file_path.h"
27 : #include "base/files/file_util.h"
28 : #include "base/files/scoped_temp_dir.h"
29 : #include "base/strings/string16.h"
30 : #include "base/strings/string_number_conversions.h"
31 : #include "base/strings/utf_string_conversions.h"
32 : #include "base/time/time.h"
33 : #include "gtest/gtest.h"
34 : #include "syzygy/kasko/crash_keys_serialization.h"
35 :
36 m : namespace kasko {
37 :
38 m : namespace {
39 :
40 : // This test harness allows us to generate reports and mock the results of
41 : // upload attempts for them. A report may be configured to succeed immediately,
42 : // succeed after 1 or 2 retries, or fail permanently.
43 : // The test harness will log failures if the permanent failure or upload
44 : // handler is invoked inappropriately or not invoked when expected.
45 : // A mock TimeSource is used to simulate the passage of time for retry
46 : // intervals.
47 : // Each test should call repository()->UploadPendingReport() enough times to
48 : // empty the repository. The harness expects it to be empty at the end of the
49 : // test.
50 m : class ReportRepositoryTest : public testing::Test {
51 m : public:
52 m : static const uint16_t kHalfRetryIntervalInSeconds;
53 m : static const uint16_t kRetryIntervalInSeconds;
54 :
55 : // The mock time must not start at 0, as we cannot update a file timestamp to
56 : // that value.
57 m : ReportRepositoryTest()
58 m : : remainder_expected_(false), time_(base::Time::Now()) {}
59 :
60 m : protected:
61 m : typedef std::pair<std::string, std::map<base::string16, base::string16>>
62 m : Report;
63 :
64 : // testing::Test implementation
65 m : void SetUp() override {
66 m : repository_temp_dir_.CreateUniqueTempDir();
67 m : repository_.reset(new ReportRepository(
68 m : repository_temp_dir_.path(),
69 m : base::TimeDelta::FromSeconds(kRetryIntervalInSeconds),
70 m : base::Bind(&ReportRepositoryTest::GetTime, base::Unretained(this)),
71 m : base::Bind(&ReportRepositoryTest::Upload, base::Unretained(this)),
72 m : base::Bind(&ReportRepositoryTest::HandlePermanentFailure,
73 m : base::Unretained(this))));
74 m : }
75 m : void TearDown() override { Validate(); }
76 :
77 : // Validates that all injected reports have been handled as expected, and that
78 : // the repository directory does not contain any leftover files.
79 : // This is automatically called by TearDown but may also be invoked mid-test.
80 m : void Validate() {
81 : // There should not be anything left over.
82 m : EXPECT_EQ(base::FilePath(),
83 m : base::FileEnumerator(repository_temp_dir_.path(), true,
84 m : base::FileEnumerator::FILES).Next());
85 :
86 : // If |remainder_expected_| is true, we allow exactly one report to not be
87 : // processed (due to corruption).
88 m : for (size_t i = 0; i < arraysize(successful_reports_); ++i) {
89 m : if (remainder_expected_ && successful_reports_[i].size() == 1) {
90 m : remainder_expected_ = false;
91 m : successful_reports_[i].clear();
92 m : continue;
93 m : }
94 m : EXPECT_TRUE(successful_reports_[i].empty());
95 m : }
96 m : for (size_t i = 0; i < arraysize(failing_reports_); ++i) {
97 m : if (remainder_expected_ && failing_reports_[i].size() == 1) {
98 m : remainder_expected_ = false;
99 m : failing_reports_[i].clear();
100 m : continue;
101 m : }
102 m : EXPECT_TRUE(failing_reports_[i].empty());
103 m : }
104 :
105 : // |remainder_expected_| should have been reset during the above loops.
106 m : EXPECT_FALSE(remainder_expected_);
107 m : }
108 :
109 : // Indicates that one report has been intentionally corrupted. This will be
110 : // checked during Validate().
111 m : void SetRemainderExpected() { remainder_expected_ = true; }
112 :
113 : // Randomly deletes a report file (either crash keys or minidump) from the
114 : // repository.
115 m : void OrphanAReport() {
116 m : base::FilePath to_delete;
117 m : size_t count = 0;
118 m : base::FileEnumerator file_enumerator(repository_temp_dir_.path(), true,
119 m : base::FileEnumerator::FILES);
120 m : for (base::FilePath candidate = file_enumerator.Next(); !candidate.empty();
121 m : candidate = file_enumerator.Next()) {
122 m : ++count;
123 m : if (base::RandDouble() < 1.0 / count)
124 m : to_delete = candidate;
125 m : }
126 m : ASSERT_FALSE(to_delete.empty());
127 m : ASSERT_TRUE(base::DeleteFile(to_delete, false));
128 m : }
129 :
130 : // Implements the TimeSource.
131 m : base::Time GetTime() { return time_; }
132 :
133 : // Increments the simulated clock.
134 m : void IncrementTime(const base::TimeDelta& time_delta) { time_ += time_delta; }
135 :
136 : // Creates a report that will succeed after the specified number of retries
137 : // (0, 1, or 2).
138 m : void InjectForSuccessAfterRetries(size_t retries) {
139 m : Report report = GenerateReport();
140 m : AllowReportToSucceedAfterRetries(report, retries);
141 m : StoreReport(report);
142 m : }
143 :
144 : // Creates a report that will never succeed in uploading.
145 m : void InjectForFailure() {
146 m : Report report = GenerateReport();
147 m : PermanentlyFailReport(report);
148 m : StoreReport(report);
149 m : }
150 :
151 : // Returns the instance under test.
152 m : ReportRepository* repository() { return repository_.get(); }
153 :
154 m : private:
155 : // Writes a report to disk and stores it in the repository.
156 m : void StoreReport(const Report& report) {
157 m : base::FilePath minidump_file;
158 m : ASSERT_TRUE(base::CreateTemporaryFileInDir(repository_temp_dir_.path(),
159 m : &minidump_file));
160 m : ASSERT_TRUE(base::WriteFile(minidump_file, report.first.data(),
161 m : report.first.length()));
162 m : repository_->StoreReport(minidump_file, report.second);
163 m : }
164 :
165 : // Sets up the mock behaviour for a report that will succeed after the
166 : // specified number of retries.
167 m : void AllowReportToSucceedAfterRetries(const Report& report, size_t retries) {
168 m : ASSERT_LT(retries, arraysize(successful_reports_));
169 m : successful_reports_[retries].push_back(report);
170 m : }
171 :
172 : // Sets up the mock behaviour for a report that will always fail to upload.
173 m : void PermanentlyFailReport(const Report& report) {
174 m : failing_reports_[3].push_back(report);
175 m : }
176 :
177 : // Generates a unique report.
178 m : Report GenerateReport() {
179 m : static size_t id = 0;
180 m : Report report;
181 m : report.first = base::UintToString(id);
182 m : report.second[L"id"] = base::ASCIIToUTF16(report.first);
183 m : ++id;
184 m : return report;
185 m : }
186 :
187 : // Implements the UploadHandler.
188 m : bool Upload(const base::FilePath& minidump_path,
189 m : const std::map<base::string16, base::string16>& crash_keys) {
190 m : Report report;
191 m : bool success = base::ReadFileToString(minidump_path, &report.first);
192 m : EXPECT_TRUE(success);
193 m : if (!success)
194 m : return false;
195 :
196 m : report.second = crash_keys;
197 :
198 : // Check to see if this report is destined to eventually succeed. If it's in
199 : // successful_reports_[0] it succeeds this round. If it's in [1] or higher
200 : // it will fail this round but be advanced to a lower index to eventually
201 : // succeed.
202 m : for (size_t i = 0; i < arraysize(successful_reports_); ++i) {
203 m : std::vector<Report>::iterator entry = std::find(
204 m : successful_reports_[i].begin(), successful_reports_[i].end(), report);
205 m : if (entry == successful_reports_[i].end())
206 m : continue;
207 :
208 : // Remove it from whence it was found.
209 m : successful_reports_[i].erase(entry);
210 : // Advance it, if necessary.
211 m : if (i > 0) {
212 m : successful_reports_[i-1].push_back(report);
213 m : return false;
214 m : }
215 m : return true;
216 m : }
217 :
218 : // Check to see if this report is destined for permanent failure.
219 : // Start at [1] because the elements in [0] are ready for
220 : // HandlePermanentFailure.
221 m : for (size_t i = 1; i < arraysize(failing_reports_); ++i) {
222 m : std::vector<Report>::iterator entry = std::find(
223 m : failing_reports_[i].begin(), failing_reports_[i].end(), report);
224 m : if (entry == failing_reports_[i].end())
225 m : continue;
226 : // Remove it from whence it was found.
227 m : failing_reports_[i].erase(entry);
228 : // Advance towards later permanent failure.
229 m : failing_reports_[i - 1].push_back(report);
230 m : return false;
231 m : }
232 m : ADD_FAILURE() << "Unexpected report. Minidump: " << report.first;
233 m : return false;
234 m : }
235 :
236 : // Implements the PermanentFailureHandler.
237 m : void HandlePermanentFailure(const base::FilePath& minidump_path,
238 m : const base::FilePath& crash_keys_path) {
239 m : Report report;
240 m : EXPECT_TRUE(ReadCrashKeysFromFile(crash_keys_path, &report.second));
241 m : ASSERT_TRUE(base::ReadFileToString(minidump_path, &report.first));
242 :
243 m : std::vector<Report>::iterator entry = std::find(
244 m : failing_reports_[0].begin(), failing_reports_[0].end(), report);
245 m : if (entry == failing_reports_[0].end()) {
246 m : ADD_FAILURE() << "Unexpected permanently failed report. Minidump: "
247 m : << report.first;
248 m : } else {
249 m : failing_reports_[0].erase(entry);
250 m : }
251 m : }
252 :
253 : // If true, exactly one report should never have been sent (because we
254 : // corrupted it).
255 m : bool remainder_expected_;
256 :
257 : // Vectors of reports that should succeed after 0, 1, or 2 failures according
258 : // to their index in this array.
259 m : std::vector<Report> successful_reports_[3];
260 :
261 : // Vectors of reports that should permanently fail after 1, 2, or 3 more
262 : // failures according to their index in this array. Index [0] is reports that
263 : // have just failed upload and should now be handed to the
264 : // PermanentFailureHandler.
265 m : std::vector<Report> failing_reports_[4];
266 :
267 : // The repository directory.
268 m : base::ScopedTempDir repository_temp_dir_;
269 :
270 : // The mock time.
271 m : base::Time time_;
272 :
273 : // The instance under test.
274 m : std::unique_ptr<ReportRepository> repository_;
275 :
276 m : DISALLOW_COPY_AND_ASSIGN(ReportRepositoryTest);
277 m : };
278 :
279 m : const uint16_t ReportRepositoryTest::kHalfRetryIntervalInSeconds = 10;
280 m : const uint16_t ReportRepositoryTest::kRetryIntervalInSeconds =
281 m : ReportRepositoryTest::kHalfRetryIntervalInSeconds * 2;
282 :
283 m : } // namespace
284 :
285 m : TEST_F(ReportRepositoryTest, BasicTest) {
286 m : EXPECT_FALSE(repository()->HasPendingReports());
287 :
288 m : InjectForSuccessAfterRetries(2);
289 m : EXPECT_TRUE(repository()->HasPendingReports());
290 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
291 m : EXPECT_FALSE(repository()->HasPendingReports());
292 :
293 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
294 :
295 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
296 m : EXPECT_TRUE(repository()->HasPendingReports());
297 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
298 m : EXPECT_FALSE(repository()->HasPendingReports());
299 :
300 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
301 m : EXPECT_TRUE(repository()->HasPendingReports());
302 m : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
303 m : EXPECT_FALSE(repository()->HasPendingReports());
304 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
305 :
306 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
307 m : EXPECT_FALSE(repository()->HasPendingReports());
308 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
309 m : }
310 :
311 m : TEST_F(ReportRepositoryTest, SuccessTest) {
312 m : EXPECT_FALSE(repository()->HasPendingReports());
313 :
314 m : InjectForSuccessAfterRetries(0);
315 m : EXPECT_TRUE(repository()->HasPendingReports());
316 m : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
317 m : EXPECT_FALSE(repository()->HasPendingReports());
318 :
319 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
320 :
321 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
322 m : EXPECT_FALSE(repository()->HasPendingReports());
323 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
324 m : }
325 :
326 m : TEST_F(ReportRepositoryTest, PermanentFailureTest) {
327 m : EXPECT_FALSE(repository()->HasPendingReports());
328 :
329 m : InjectForFailure();
330 m : EXPECT_TRUE(repository()->HasPendingReports());
331 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
332 m : EXPECT_FALSE(repository()->HasPendingReports());
333 :
334 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
335 m : EXPECT_TRUE(repository()->HasPendingReports());
336 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
337 m : EXPECT_FALSE(repository()->HasPendingReports());
338 :
339 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
340 m : EXPECT_TRUE(repository()->HasPendingReports());
341 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Fails
342 m : EXPECT_FALSE(repository()->HasPendingReports());
343 m : }
344 :
345 m : TEST_F(ReportRepositoryTest, MultipleReportsTest) {
346 m : EXPECT_FALSE(repository()->HasPendingReports());
347 :
348 m : InjectForSuccessAfterRetries(0);
349 m : InjectForSuccessAfterRetries(0);
350 m : InjectForSuccessAfterRetries(0);
351 :
352 m : EXPECT_TRUE(repository()->HasPendingReports());
353 m : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
354 m : EXPECT_TRUE(repository()->HasPendingReports());
355 m : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
356 m : EXPECT_TRUE(repository()->HasPendingReports());
357 m : EXPECT_TRUE(repository()->UploadPendingReport()); // Succeeds
358 m : EXPECT_FALSE(repository()->HasPendingReports());
359 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
360 :
361 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
362 m : EXPECT_FALSE(repository()->HasPendingReports());
363 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
364 m : }
365 :
366 m : TEST_F(ReportRepositoryTest, MultipleReportsTestWithFailures) {
367 m : EXPECT_FALSE(repository()->HasPendingReports());
368 :
369 m : InjectForSuccessAfterRetries(0);
370 m : InjectForSuccessAfterRetries(1);
371 m : InjectForSuccessAfterRetries(2);
372 m : InjectForFailure();
373 :
374 : // 3 will fail, 1 will succeed.
375 m : size_t successes = 0;
376 m : for (size_t i = 0; i < 4; ++i) {
377 m : EXPECT_TRUE(repository()->HasPendingReports());
378 m : if (repository()->UploadPendingReport())
379 m : successes++;
380 m : }
381 m : EXPECT_EQ(1, successes);
382 m : EXPECT_FALSE(repository()->HasPendingReports());
383 m : EXPECT_TRUE(repository()->UploadPendingReport());
384 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
385 :
386 : // 2 will fail, 1 will succeed.
387 m : successes = 0;
388 m : for (size_t i = 0; i < 3; ++i) {
389 m : EXPECT_TRUE(repository()->HasPendingReports());
390 m : if (repository()->UploadPendingReport())
391 m : successes++;
392 m : }
393 m : EXPECT_EQ(1, successes);
394 m : EXPECT_FALSE(repository()->HasPendingReports());
395 m : EXPECT_TRUE(repository()->UploadPendingReport());
396 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
397 :
398 : // 1 will permanently fail, 1 will succeed.
399 m : successes = 0;
400 m : for (size_t i = 0; i < 2; ++i) {
401 m : EXPECT_TRUE(repository()->HasPendingReports());
402 m : if (repository()->UploadPendingReport())
403 m : successes++;
404 m : }
405 m : EXPECT_EQ(1, successes);
406 m : EXPECT_FALSE(repository()->HasPendingReports());
407 m : EXPECT_TRUE(repository()->UploadPendingReport());
408 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
409 :
410 : // None left.
411 m : EXPECT_FALSE(repository()->HasPendingReports());
412 m : EXPECT_TRUE(repository()->UploadPendingReport());
413 m : }
414 :
415 m : TEST_F(ReportRepositoryTest, MultipleInterleavedReports) {
416 m : EXPECT_FALSE(repository()->HasPendingReports());
417 :
418 : // 1st generation
419 m : InjectForSuccessAfterRetries(1);
420 m : InjectForSuccessAfterRetries(2);
421 :
422 m : EXPECT_TRUE(repository()->HasPendingReports());
423 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
424 m : EXPECT_TRUE(repository()->HasPendingReports());
425 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
426 m : EXPECT_FALSE(repository()->HasPendingReports());
427 :
428 : // Increment a half interval.
429 m : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
430 m : EXPECT_FALSE(repository()->HasPendingReports());
431 : // 2nd generation
432 m : InjectForSuccessAfterRetries(1);
433 m : InjectForSuccessAfterRetries(2);
434 m : EXPECT_TRUE(repository()->HasPendingReports());
435 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
436 m : EXPECT_TRUE(repository()->HasPendingReports());
437 m : EXPECT_FALSE(repository()->UploadPendingReport()); // Failure
438 m : EXPECT_FALSE(repository()->HasPendingReports());
439 m : EXPECT_TRUE(repository()->UploadPendingReport()); // No-op
440 :
441 : // Increment another half interval. Now only the first generation are eligible
442 : // for retry. One will succeed.
443 m : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
444 m : size_t successes = 0;
445 m : for (size_t i = 0; i < 2; ++i) {
446 m : EXPECT_TRUE(repository()->HasPendingReports());
447 m : if (repository()->UploadPendingReport())
448 m : successes++;
449 m : }
450 m : EXPECT_EQ(1, successes);
451 m : EXPECT_FALSE(repository()->HasPendingReports());
452 :
453 : // Increment another half interval. This is the second generation, one will
454 : // succeed.
455 m : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
456 m : successes = 0;
457 m : for (size_t i = 0; i < 2; ++i) {
458 m : EXPECT_TRUE(repository()->HasPendingReports());
459 m : if (repository()->UploadPendingReport())
460 m : successes++;
461 m : }
462 m : EXPECT_EQ(1, successes);
463 m : EXPECT_FALSE(repository()->HasPendingReports());
464 :
465 : // Increment another half interval. This is the first generation, only one
466 : // element left (it will succeed).
467 m : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
468 m : EXPECT_TRUE(repository()->HasPendingReports());
469 m : EXPECT_TRUE(repository()->UploadPendingReport());
470 m : EXPECT_FALSE(repository()->HasPendingReports());
471 :
472 : // Increment another half interval. This is the second generation, only one
473 : // element left (it will succeed).
474 m : IncrementTime(base::TimeDelta::FromSeconds(kHalfRetryIntervalInSeconds));
475 m : EXPECT_TRUE(repository()->HasPendingReports());
476 m : EXPECT_TRUE(repository()->UploadPendingReport());
477 m : EXPECT_FALSE(repository()->HasPendingReports());
478 m : }
479 :
480 m : TEST_F(ReportRepositoryTest, CorruptionTest) {
481 : // In order to avoid hard-coding extensions/paths, and having a bunch of
482 : // permutations, let's run this test a bunch of times and probabilistically
483 : // cover all the cases of a file being missing.
484 m : for (size_t i = 0; i< 100; ++i) {
485 : // This sequence will put one report each in the different states.
486 m : InjectForSuccessAfterRetries(2); // one in Incoming
487 m : InjectForSuccessAfterRetries(2); // two in Incoming
488 m : repository()->UploadPendingReport(); // one in Retry
489 m : repository()->UploadPendingReport(); // two in Retry
490 m : IncrementTime(base::TimeDelta::FromSeconds(kRetryIntervalInSeconds));
491 m : repository()->UploadPendingReport(); // one in Retry 2
492 m : InjectForSuccessAfterRetries(2); // one in Incoming
493 :
494 : // Randomly delete one file.
495 m : OrphanAReport();
496 :
497 : // Wait 36 hours.
498 m : base::Time start = GetTime();
499 m : while (GetTime() - start < base::TimeDelta::FromHours(36)) {
500 m : IncrementTime(base::TimeDelta::FromMinutes(30));
501 m : repository()->UploadPendingReport();
502 m : }
503 :
504 m : SetRemainderExpected();
505 : // Validate that exactly one of the injected reports didn't come out and
506 : // that there are no files left over.
507 m : Validate();
508 m : }
509 m : }
510 :
511 m : } // namespace kasko
|