backupable_db_test.cc 54.2 KB
Newer Older
1
//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
Siying Dong's avatar
Siying Dong committed
2
3
4
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
Igor Canadi's avatar
Igor Canadi committed
5
6
7
8
9
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

10
#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
11

12
#include <algorithm>
13
#include <string>
14

15
#include "db/db_impl.h"
16
#include "env/env_chroot.h"
Igor Canadi's avatar
Igor Canadi committed
17
#include "port/port.h"
18
#include "port/stack_trace.h"
19
#include "rocksdb/rate_limiter.h"
Igor Canadi's avatar
Igor Canadi committed
20
#include "rocksdb/transaction_log.h"
21
#include "rocksdb/types.h"
22
#include "rocksdb/utilities/backupable_db.h"
Wanning Jiang's avatar
Wanning Jiang committed
23
#include "rocksdb/utilities/options_util.h"
24
#include "util/file_reader_writer.h"
25
#include "util/filename.h"
Igor Canadi's avatar
Igor Canadi committed
26
#include "util/mutexlock.h"
27
#include "util/random.h"
Wanning Jiang's avatar
Wanning Jiang committed
28
#include "util/stderr_logger.h"
sdong's avatar
sdong committed
29
#include "util/string_util.h"
30
#include "util/sync_point.h"
31
#include "util/testharness.h"
Igor Canadi's avatar
Igor Canadi committed
32
33
34
35
36
37
38
39
40
41
42
43
44
#include "util/testutil.h"

namespace rocksdb {

namespace {

using std::unique_ptr;

class DummyDB : public StackableDB {
 public:
  /* implicit */
  DummyDB(const Options& options, const std::string& dbname)
     : StackableDB(nullptr), options_(options), dbname_(dbname),
45
46
       deletions_enabled_(true), sequence_number_(0) {}

Igor Sugak's avatar
Igor Sugak committed
47
  virtual SequenceNumber GetLatestSequenceNumber() const override {
48
49
    return ++sequence_number_;
  }
Igor Canadi's avatar
Igor Canadi committed
50
51
52
53
54
55
56
57
58

  virtual const std::string& GetName() const override {
    return dbname_;
  }

  virtual Env* GetEnv() const override {
    return options_.env;
  }

59
  using DB::GetOptions;
Andrew Kryczka's avatar
Andrew Kryczka committed
60
61
  virtual Options GetOptions(
      ColumnFamilyHandle* /*column_family*/) const override {
Igor Canadi's avatar
Igor Canadi committed
62
63
64
    return options_;
  }

65
66
67
68
  virtual DBOptions GetDBOptions() const override {
    return DBOptions(options_);
  }

Andrew Kryczka's avatar
Andrew Kryczka committed
69
  virtual Status EnableFileDeletions(bool /*force*/) override {
70
    EXPECT_TRUE(!deletions_enabled_);
Igor Canadi's avatar
Igor Canadi committed
71
72
73
74
75
    deletions_enabled_ = true;
    return Status::OK();
  }

  virtual Status DisableFileDeletions() override {
76
    EXPECT_TRUE(deletions_enabled_);
Igor Canadi's avatar
Igor Canadi committed
77
78
79
80
81
    deletions_enabled_ = false;
    return Status::OK();
  }

  virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
Andrew Kryczka's avatar
Andrew Kryczka committed
82
                              bool /*flush_memtable*/ = true) override {
83
    EXPECT_TRUE(!deletions_enabled_);
Igor Canadi's avatar
Igor Canadi committed
84
85
86
87
88
    vec = live_files_;
    *mfs = 100;
    return Status::OK();
  }

89
90
91
92
  virtual ColumnFamilyHandle* DefaultColumnFamily() const override {
    return nullptr;
  }

Igor Canadi's avatar
Igor Canadi committed
93
94
95
96
97
98
99
100
101
102
  class DummyLogFile : public LogFile {
   public:
    /* implicit */
     DummyLogFile(const std::string& path, bool alive = true)
         : path_(path), alive_(alive) {}

    virtual std::string PathName() const override {
      return path_;
    }

Igor Sugak's avatar
Igor Sugak committed
103
    virtual uint64_t LogNumber() const override {
Igor Canadi's avatar
Igor Canadi committed
104
      // what business do you have calling this method?
105
      ADD_FAILURE();
Igor Canadi's avatar
Igor Canadi committed
106
107
108
109
110
111
112
      return 0;
    }

    virtual WalFileType Type() const override {
      return alive_ ? kAliveLogFile : kArchivedLogFile;
    }

Igor Sugak's avatar
Igor Sugak committed
113
    virtual SequenceNumber StartSequence() const override {
114
115
116
      // this seqnum guarantees the dummy file will be included in the backup
      // as long as it is alive.
      return kMaxSequenceNumber;
Igor Canadi's avatar
Igor Canadi committed
117
118
    }

Igor Sugak's avatar
Igor Sugak committed
119
    virtual uint64_t SizeFileBytes() const override {
Igor Canadi's avatar
Igor Canadi committed
120
121
122
123
124
125
126
127
128
      return 0;
    }

   private:
    std::string path_;
    bool alive_;
  }; // DummyLogFile

  virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
129
    EXPECT_TRUE(!deletions_enabled_);
Igor Canadi's avatar
Igor Canadi committed
130
131
132
133
134
135
136
137
    files.resize(wal_files_.size());
    for (size_t i = 0; i < files.size(); ++i) {
      files[i].reset(
          new DummyLogFile(wal_files_[i].first, wal_files_[i].second));
    }
    return Status::OK();
  }

138
  // To avoid FlushWAL called on stacked db which is nullptr
Andrew Kryczka's avatar
Andrew Kryczka committed
139
  virtual Status FlushWAL(bool /*sync*/) override { return Status::OK(); }
140

Igor Canadi's avatar
Igor Canadi committed
141
142
143
144
145
146
147
  std::vector<std::string> live_files_;
  // pair<filename, alive?>
  std::vector<std::pair<std::string, bool>> wal_files_;
 private:
  Options options_;
  std::string dbname_;
  bool deletions_enabled_;
148
  mutable SequenceNumber sequence_number_;
Igor Canadi's avatar
Igor Canadi committed
149
150
151
152
153
154
155
}; // DummyDB

class TestEnv : public EnvWrapper {
 public:
  explicit TestEnv(Env* t) : EnvWrapper(t) {}

  class DummySequentialFile : public SequentialFile {
156
   public:
157
158
    explicit DummySequentialFile(bool fail_reads)
        : SequentialFile(), rnd_(5), fail_reads_(fail_reads) {}
Igor Sugak's avatar
Igor Sugak committed
159
    virtual Status Read(size_t n, Slice* result, char* scratch) override {
160
161
162
      if (fail_reads_) {
        return Status::IOError();
      }
Igor Canadi's avatar
Igor Canadi committed
163
      size_t read_size = (n > size_left) ? size_left : n;
164
165
166
      for (size_t i = 0; i < read_size; ++i) {
        scratch[i] = rnd_.Next() & 255;
      }
Igor Canadi's avatar
Igor Canadi committed
167
168
169
170
171
      *result = Slice(scratch, read_size);
      size_left -= read_size;
      return Status::OK();
    }

Igor Sugak's avatar
Igor Sugak committed
172
    virtual Status Skip(uint64_t n) override {
Igor Canadi's avatar
Igor Canadi committed
173
174
175
176
177
      size_left = (n > size_left) ? size_left - n : 0;
      return Status::OK();
    }
   private:
    size_t size_left = 200;
178
    Random rnd_;
179
    bool fail_reads_;
Igor Canadi's avatar
Igor Canadi committed
180
181
  };

Igor Sugak's avatar
Igor Sugak committed
182
183
  Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
                           const EnvOptions& options) override {
Igor Canadi's avatar
Igor Canadi committed
184
    MutexLock l(&mutex_);
Igor Canadi's avatar
Igor Canadi committed
185
    if (dummy_sequential_file_) {
186
187
      r->reset(
          new TestEnv::DummySequentialFile(dummy_sequential_file_fail_reads_));
Igor Canadi's avatar
Igor Canadi committed
188
189
190
191
192
193
194
      return Status::OK();
    } else {
      return EnvWrapper::NewSequentialFile(f, r, options);
    }
  }

  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
Igor Sugak's avatar
Igor Sugak committed
195
                         const EnvOptions& options) override {
Igor Canadi's avatar
Igor Canadi committed
196
    MutexLock l(&mutex_);
Lei Jin's avatar
Lei Jin committed
197
    written_files_.push_back(f);
Igor Canadi's avatar
Igor Canadi committed
198
    if (limit_written_files_ <= 0) {
Lei Jin's avatar
Lei Jin committed
199
      return Status::NotSupported("Sorry, can't do this");
Igor Canadi's avatar
Igor Canadi committed
200
201
202
203
204
    }
    limit_written_files_--;
    return EnvWrapper::NewWritableFile(f, r, options);
  }

Igor Canadi's avatar
Igor Canadi committed
205
  virtual Status DeleteFile(const std::string& fname) override {
Igor Canadi's avatar
Igor Canadi committed
206
    MutexLock l(&mutex_);
207
208
209
    if (fail_delete_files_) {
      return Status::IOError();
    }
210
    EXPECT_GT(limit_delete_files_, 0U);
Igor Canadi's avatar
Igor Canadi committed
211
212
213
214
    limit_delete_files_--;
    return EnvWrapper::DeleteFile(fname);
  }

215
216
217
218
219
220
221
222
  virtual Status DeleteDir(const std::string& dirname) override {
    MutexLock l(&mutex_);
    if (fail_delete_files_) {
      return Status::IOError();
    }
    return EnvWrapper::DeleteDir(dirname);
  }

Lei Jin's avatar
Lei Jin committed
223
  void AssertWrittenFiles(std::vector<std::string>& should_have_written) {
Igor Canadi's avatar
Igor Canadi committed
224
    MutexLock l(&mutex_);
225
226
    std::sort(should_have_written.begin(), should_have_written.end());
    std::sort(written_files_.begin(), written_files_.end());
Wanning Jiang's avatar
Wanning Jiang committed
227

Andrew Kryczka's avatar
Andrew Kryczka committed
228
    ASSERT_EQ(should_have_written, written_files_);
Igor Canadi's avatar
Igor Canadi committed
229
230
  }

Lei Jin's avatar
Lei Jin committed
231
  void ClearWrittenFiles() {
Igor Canadi's avatar
Igor Canadi committed
232
    MutexLock l(&mutex_);
Lei Jin's avatar
Lei Jin committed
233
    written_files_.clear();
Igor Canadi's avatar
Igor Canadi committed
234
235
236
  }

  void SetLimitWrittenFiles(uint64_t limit) {
Igor Canadi's avatar
Igor Canadi committed
237
    MutexLock l(&mutex_);
Igor Canadi's avatar
Igor Canadi committed
238
239
240
    limit_written_files_ = limit;
  }

Igor Canadi's avatar
Igor Canadi committed
241
242
243
244
  void SetLimitDeleteFiles(uint64_t limit) {
    MutexLock l(&mutex_);
    limit_delete_files_ = limit;
  }
Igor Canadi's avatar
Igor Canadi committed
245

246
247
248
249
250
  void SetDeleteFileFailure(bool fail) {
    MutexLock l(&mutex_);
    fail_delete_files_ = fail;
  }

Igor Canadi's avatar
Igor Canadi committed
251
  void SetDummySequentialFile(bool dummy_sequential_file) {
Igor Canadi's avatar
Igor Canadi committed
252
    MutexLock l(&mutex_);
Igor Canadi's avatar
Igor Canadi committed
253
254
    dummy_sequential_file_ = dummy_sequential_file;
  }
255
256
257
258
  void SetDummySequentialFileFailReads(bool dummy_sequential_file_fail_reads) {
    MutexLock l(&mutex_);
    dummy_sequential_file_fail_reads_ = dummy_sequential_file_fail_reads;
  }
Igor Canadi's avatar
Igor Canadi committed
259

260
261
262
263
264
265
266
267
268
  void SetGetChildrenFailure(bool fail) { get_children_failure_ = fail; }
  Status GetChildren(const std::string& dir,
                     std::vector<std::string>* r) override {
    if (get_children_failure_) {
      return Status::IOError("SimulatedFailure");
    }
    return EnvWrapper::GetChildren(dir, r);
  }

269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
  // Some test cases do not actually create the test files (e.g., see
  // DummyDB::live_files_) - for those cases, we mock those files' attributes
  // so CreateNewBackup() can get their attributes.
  void SetFilenamesForMockedAttrs(const std::vector<std::string>& filenames) {
    filenames_for_mocked_attrs_ = filenames;
  }
  Status GetChildrenFileAttributes(
      const std::string& dir, std::vector<Env::FileAttributes>* r) override {
    if (filenames_for_mocked_attrs_.size() > 0) {
      for (const auto& filename : filenames_for_mocked_attrs_) {
        r->push_back({dir + filename, 10 /* size_bytes */});
      }
      return Status::OK();
    }
    return EnvWrapper::GetChildrenFileAttributes(dir, r);
  }
285
286
287
288
289
290
291
292
293
294
295
296
297
  Status GetFileSize(const std::string& path, uint64_t* size_bytes) override {
    if (filenames_for_mocked_attrs_.size() > 0) {
      auto fname = path.substr(path.find_last_of('/'));
      auto filename_iter = std::find(filenames_for_mocked_attrs_.begin(),
                                     filenames_for_mocked_attrs_.end(), fname);
      if (filename_iter != filenames_for_mocked_attrs_.end()) {
        *size_bytes = 10;
        return Status::OK();
      }
      return Status::NotFound(fname);
    }
    return EnvWrapper::GetFileSize(path, size_bytes);
  }
298

299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
  void SetCreateDirIfMissingFailure(bool fail) {
    create_dir_if_missing_failure_ = fail;
  }
  Status CreateDirIfMissing(const std::string& d) override {
    if (create_dir_if_missing_failure_) {
      return Status::IOError("SimulatedFailure");
    }
    return EnvWrapper::CreateDirIfMissing(d);
  }

  void SetNewDirectoryFailure(bool fail) { new_directory_failure_ = fail; }
  virtual Status NewDirectory(const std::string& name,
                              unique_ptr<Directory>* result) override {
    if (new_directory_failure_) {
      return Status::IOError("SimulatedFailure");
    }
    return EnvWrapper::NewDirectory(name, result);
  }

Igor Canadi's avatar
Igor Canadi committed
318
 private:
Igor Canadi's avatar
Igor Canadi committed
319
  port::Mutex mutex_;
Igor Canadi's avatar
Igor Canadi committed
320
  bool dummy_sequential_file_ = false;
321
  bool dummy_sequential_file_fail_reads_ = false;
Lei Jin's avatar
Lei Jin committed
322
  std::vector<std::string> written_files_;
323
  std::vector<std::string> filenames_for_mocked_attrs_;
Igor Canadi's avatar
Igor Canadi committed
324
  uint64_t limit_written_files_ = 1000000;
Igor Canadi's avatar
Igor Canadi committed
325
  uint64_t limit_delete_files_ = 1000000;
326
  bool fail_delete_files_ = false;
327
328
329
330

  bool get_children_failure_ = false;
  bool create_dir_if_missing_failure_ = false;
  bool new_directory_failure_ = false;
Igor Canadi's avatar
Igor Canadi committed
331
};  // TestEnv
Igor Canadi's avatar
Igor Canadi committed
332
333
334
335
336

class FileManager : public EnvWrapper {
 public:
  explicit FileManager(Env* t) : EnvWrapper(t), rnd_(5) {}

337
  Status DeleteRandomFileInDir(const std::string& dir) {
Igor Canadi's avatar
Igor Canadi committed
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
    std::vector<std::string> children;
    GetChildren(dir, &children);
    if (children.size() <= 2) { // . and ..
      return Status::NotFound("");
    }
    while (true) {
      int i = rnd_.Next() % children.size();
      if (children[i] != "." && children[i] != "..") {
        return DeleteFile(dir + "/" + children[i]);
      }
    }
    // should never get here
    assert(false);
    return Status::NotFound("");
  }

354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
  Status AppendToRandomFileInDir(const std::string& dir,
                                 const std::string& data) {
    std::vector<std::string> children;
    GetChildren(dir, &children);
    if (children.size() <= 2) {
      return Status::NotFound("");
    }
    while (true) {
      int i = rnd_.Next() % children.size();
      if (children[i] != "." && children[i] != "..") {
        return WriteToFile(dir + "/" + children[i], data);
      }
    }
    // should never get here
    assert(false);
    return Status::NotFound("");
  }

Igor Canadi's avatar
Igor Canadi committed
372
  Status CorruptFile(const std::string& fname, uint64_t bytes_to_corrupt) {
373
374
    std::string file_contents;
    Status s = ReadFileToString(this, fname, &file_contents);
Igor Canadi's avatar
Igor Canadi committed
375
376
377
    if (!s.ok()) {
      return s;
    }
378
    s = DeleteFile(fname);
Igor Canadi's avatar
Igor Canadi committed
379
380
381
    if (!s.ok()) {
      return s;
    }
382
383

    for (uint64_t i = 0; i < bytes_to_corrupt; ++i) {
Igor Canadi's avatar
Igor Canadi committed
384
      std::string tmp;
385
386
      test::RandomString(&rnd_, 1, &tmp);
      file_contents[rnd_.Next() % file_contents.size()] = tmp[0];
Igor Canadi's avatar
Igor Canadi committed
387
    }
388
    return WriteToFile(fname, file_contents);
Igor Canadi's avatar
Igor Canadi committed
389
390
  }

Lei Jin's avatar
Lei Jin committed
391
392
393
394
395
396
397
398
399
400
401
  Status CorruptChecksum(const std::string& fname, bool appear_valid) {
    std::string metadata;
    Status s = ReadFileToString(this, fname, &metadata);
    if (!s.ok()) {
      return s;
    }
    s = DeleteFile(fname);
    if (!s.ok()) {
      return s;
    }

402
403
404
405
406
    auto pos = metadata.find("private");
    if (pos == std::string::npos) {
      return Status::Corruption("private file is expected");
    }
    pos = metadata.find(" crc32 ", pos + 6);
Lei Jin's avatar
Lei Jin committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
    if (pos == std::string::npos) {
      return Status::Corruption("checksum not found");
    }

    if (metadata.size() < pos + 7) {
      return Status::Corruption("bad CRC32 checksum value");
    }

    if (appear_valid) {
      if (metadata[pos + 8] == '\n') {
        // single digit value, safe to insert one more digit
        metadata.insert(pos + 8, 1, '0');
      } else {
        metadata.erase(pos + 8, 1);
      }
    } else {
      metadata[pos + 7] = 'a';
    }

    return WriteToFile(fname, metadata);
  }

Igor Canadi's avatar
Igor Canadi committed
429
430
431
432
433
434
435
436
437
438
  Status WriteToFile(const std::string& fname, const std::string& data) {
    unique_ptr<WritableFile> file;
    EnvOptions env_options;
    env_options.use_mmap_writes = false;
    Status s = EnvWrapper::NewWritableFile(fname, &file, env_options);
    if (!s.ok()) {
      return s;
    }
    return file->Append(Slice(data));
  }
Lei Jin's avatar
Lei Jin committed
439

Igor Canadi's avatar
Igor Canadi committed
440
441
442
443
444
 private:
  Random rnd_;
}; // FileManager

// utility functions
Igor Canadi's avatar
Igor Canadi committed
445
446
static size_t FillDB(DB* db, int from, int to) {
  size_t bytes_written = 0;
Igor Canadi's avatar
Igor Canadi committed
447
  for (int i = from; i < to; ++i) {
sdong's avatar
sdong committed
448
449
    std::string key = "testkey" + ToString(i);
    std::string value = "testvalue" + ToString(i);
Igor Canadi's avatar
Igor Canadi committed
450
    bytes_written += key.size() + value.size();
Igor Canadi's avatar
Igor Canadi committed
451

452
    EXPECT_OK(db->Put(WriteOptions(), Slice(key), Slice(value)));
Igor Canadi's avatar
Igor Canadi committed
453
  }
Igor Canadi's avatar
Igor Canadi committed
454
  return bytes_written;
Igor Canadi's avatar
Igor Canadi committed
455
456
457
458
}

static void AssertExists(DB* db, int from, int to) {
  for (int i = from; i < to; ++i) {
sdong's avatar
sdong committed
459
    std::string key = "testkey" + ToString(i);
Igor Canadi's avatar
Igor Canadi committed
460
461
    std::string value;
    Status s = db->Get(ReadOptions(), Slice(key), &value);
sdong's avatar
sdong committed
462
    ASSERT_EQ(value, "testvalue" + ToString(i));
Igor Canadi's avatar
Igor Canadi committed
463
464
465
466
467
  }
}

static void AssertEmpty(DB* db, int from, int to) {
  for (int i = from; i < to; ++i) {
sdong's avatar
sdong committed
468
469
    std::string key = "testkey" + ToString(i);
    std::string value = "testvalue" + ToString(i);
Igor Canadi's avatar
Igor Canadi committed
470
471
472
473
474
475

    Status s = db->Get(ReadOptions(), Slice(key), &value);
    ASSERT_TRUE(s.IsNotFound());
  }
}

Igor Sugak's avatar
Igor Sugak committed
476
class BackupableDBTest : public testing::Test {
Igor Canadi's avatar
Igor Canadi committed
477
478
479
 public:
  BackupableDBTest() {
    // set up files
480
481
482
483
484
485
    std::string db_chroot = test::TmpDir() + "/backupable_db";
    std::string backup_chroot = test::TmpDir() + "/backupable_db_backup";
    Env::Default()->CreateDir(db_chroot);
    Env::Default()->CreateDir(backup_chroot);
    dbname_ = "/tempdb";
    backupdir_ = "/tempbk";
Igor Canadi's avatar
Igor Canadi committed
486
487

    // set up envs
488
489
490
491
492
    db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
    backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
    test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
    test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
    file_manager_.reset(new FileManager(backup_chroot_env_.get()));
Igor Canadi's avatar
Igor Canadi committed
493
494
495
496

    // set up db options
    options_.create_if_missing = true;
    options_.paranoid_checks = true;
497
    options_.write_buffer_size = 1 << 17; // 128KB
Igor Canadi's avatar
Igor Canadi committed
498
499
    options_.env = test_db_env_.get();
    options_.wal_dir = dbname_;
500
501
502

    // Create logger
    DBOptions logger_options;
503
    logger_options.env = db_chroot_env_.get();
504
505
    CreateLoggerFromOptions(dbname_, logger_options, &logger_);

Igor Canadi's avatar
Igor Canadi committed
506
507
    // set up backup db options
    backupable_options_.reset(new BackupableDBOptions(
508
        backupdir_, test_backup_env_.get(), true, logger_.get(), true));
Igor Canadi's avatar
Igor Canadi committed
509

510
511
512
    // most tests will use multi-threaded backups
    backupable_options_->max_background_operations = 7;

Igor Canadi's avatar
Igor Canadi committed
513
    // delete old files in db
514
    DestroyDB(dbname_, options_);
Igor Canadi's avatar
Igor Canadi committed
515
516
  }

517
518
  DB* OpenDB() {
    DB* db;
519
    EXPECT_OK(DB::Open(options_, dbname_, &db));
520
521
522
    return db;
  }

523
524
  void OpenDBAndBackupEngineShareWithChecksum(
      bool destroy_old_data = false, bool dummy = false,
Andrew Kryczka's avatar
Andrew Kryczka committed
525
      bool /*share_table_files*/ = true, bool share_with_checksums = false) {
526
527
528
529
    backupable_options_->share_files_with_checksum = share_with_checksums;
    OpenDBAndBackupEngine(destroy_old_data, dummy, share_with_checksums);
  }

530
  void OpenDBAndBackupEngine(bool destroy_old_data = false, bool dummy = false,
531
                             bool share_table_files = true) {
Igor Canadi's avatar
Igor Canadi committed
532
533
534
535
536
537
538
539
540
541
542
543
    // reset all the defaults
    test_backup_env_->SetLimitWrittenFiles(1000000);
    test_db_env_->SetLimitWrittenFiles(1000000);
    test_db_env_->SetDummySequentialFile(dummy);

    DB* db;
    if (dummy) {
      dummy_db_ = new DummyDB(options_, dbname_);
      db = dummy_db_;
    } else {
      ASSERT_OK(DB::Open(options_, dbname_, &db));
    }
544
    db_.reset(db);
Igor Canadi's avatar
Igor Canadi committed
545
    backupable_options_->destroy_old_data = destroy_old_data;
546
    backupable_options_->share_table_files = share_table_files;
547
548
549
550
    BackupEngine* backup_engine;
    ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
                                 &backup_engine));
    backup_engine_.reset(backup_engine);
Igor Canadi's avatar
Igor Canadi committed
551
552
  }

553
554
555
  void CloseDBAndBackupEngine() {
    db_.reset();
    backup_engine_.reset();
Igor Canadi's avatar
Igor Canadi committed
556
557
  }

558
  void OpenBackupEngine() {
Igor Canadi's avatar
Igor Canadi committed
559
    backupable_options_->destroy_old_data = false;
560
561
562
563
    BackupEngine* backup_engine;
    ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_,
                                 &backup_engine));
    backup_engine_.reset(backup_engine);
Igor Canadi's avatar
Igor Canadi committed
564
565
  }

566
  void CloseBackupEngine() { backup_engine_.reset(nullptr); }
Igor Canadi's avatar
Igor Canadi committed
567
568
569
570
571
572
573
574

  // restores backup backup_id and asserts the existence of
  // [start_exist, end_exist> and not-existence of
  // [end_exist, end>
  //
  // if backup_id == 0, it means restore from latest
  // if end == 0, don't check AssertEmpty
  void AssertBackupConsistency(BackupID backup_id, uint32_t start_exist,
575
576
577
                               uint32_t end_exist, uint32_t end = 0,
                               bool keep_log_files = false) {
    RestoreOptions restore_options(keep_log_files);
578
579
580
581
    bool opened_backup_engine = false;
    if (backup_engine_.get() == nullptr) {
      opened_backup_engine = true;
      OpenBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
582
583
    }
    if (backup_id > 0) {
584
585
      ASSERT_OK(backup_engine_->RestoreDBFromBackup(backup_id, dbname_, dbname_,
                                                    restore_options));
Igor Canadi's avatar
Igor Canadi committed
586
    } else {
587
588
      ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_,
                                                          restore_options));
Igor Canadi's avatar
Igor Canadi committed
589
    }
590
591
    DB* db = OpenDB();
    AssertExists(db, start_exist, end_exist);
Igor Canadi's avatar
Igor Canadi committed
592
    if (end != 0) {
593
      AssertEmpty(db, end_exist, end);
Igor Canadi's avatar
Igor Canadi committed
594
    }
595
    delete db;
596
597
    if (opened_backup_engine) {
      CloseBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
598
599
600
    }
  }

601
602
  void DeleteLogFiles() {
    std::vector<std::string> delete_logs;
603
    db_chroot_env_->GetChildren(dbname_, &delete_logs);
604
605
606
607
608
    for (auto f : delete_logs) {
      uint64_t number;
      FileType type;
      bool ok = ParseFileName(f, &number, &type);
      if (ok && type == kLogFile) {
609
        db_chroot_env_->DeleteFile(dbname_ + "/" + f);
610
611
612
613
      }
    }
  }

Igor Canadi's avatar
Igor Canadi committed
614
615
616
617
  // files
  std::string dbname_;
  std::string backupdir_;

618
619
620
621
  // logger_ must be above backup_engine_ such that the engine's destructor,
  // which uses a raw pointer to the logger, executes first.
  std::shared_ptr<Logger> logger_;

Igor Canadi's avatar
Igor Canadi committed
622
  // envs
623
624
  unique_ptr<Env> db_chroot_env_;
  unique_ptr<Env> backup_chroot_env_;
Igor Canadi's avatar
Igor Canadi committed
625
626
627
628
629
630
  unique_ptr<TestEnv> test_db_env_;
  unique_ptr<TestEnv> test_backup_env_;
  unique_ptr<FileManager> file_manager_;

  // all the dbs!
  DummyDB* dummy_db_; // BackupableDB owns dummy_db_
631
632
  unique_ptr<DB> db_;
  unique_ptr<BackupEngine> backup_engine_;
Igor Canadi's avatar
Igor Canadi committed
633
634
635

  // options
  Options options_;
636
637
638

 protected:
  unique_ptr<BackupableDBOptions> backupable_options_;
Igor Canadi's avatar
Igor Canadi committed
639
640
641
642
643
644
645
646
}; // BackupableDBTest

void AppendPath(const std::string& path, std::vector<std::string>& v) {
  for (auto& f : v) {
    f = path + f;
  }
}

647
648
649
650
class BackupableDBTestWithParam : public BackupableDBTest,
                                  public testing::WithParamInterface<bool> {
 public:
  BackupableDBTestWithParam() {
651
    backupable_options_->share_files_with_checksum = GetParam();
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
  }
};

// This test verifies that the verifyBackup method correctly identifies
// invalid backups
TEST_P(BackupableDBTestWithParam, VerifyBackup) {
  const int keys_iteration = 5000;
  Random rnd(6);
  Status s;
  OpenDBAndBackupEngine(true);
  // create five backups
  for (int i = 0; i < 5; ++i) {
    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
    ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
  }
  CloseDBAndBackupEngine();

  OpenDBAndBackupEngine();
  // ---------- case 1. - valid backup -----------
  ASSERT_TRUE(backup_engine_->VerifyBackup(1).ok());

  // ---------- case 2. - delete a file -----------i
  file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/1");
  ASSERT_TRUE(backup_engine_->VerifyBackup(1).IsNotFound());

  // ---------- case 3. - corrupt a file -----------
  std::string append_data = "Corrupting a random file";
  file_manager_->AppendToRandomFileInDir(backupdir_ + "/private/2",
                                         append_data);
  ASSERT_TRUE(backup_engine_->VerifyBackup(2).IsCorruption());

  // ---------- case 4. - invalid backup -----------
  ASSERT_TRUE(backup_engine_->VerifyBackup(6).IsNotFound());
  CloseDBAndBackupEngine();
}

// open DB, write, close DB, backup, restore, repeat
TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
  // has to be a big number, so that it triggers the memtable flush
  const int keys_iteration = 5000;
  const int max_key = keys_iteration * 4 + 10;
  // first iter -- flush before backup
  // second iter -- don't flush before backup
  for (int iter = 0; iter < 2; ++iter) {
    // delete old data
697
    DestroyDB(dbname_, options_);
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
    bool destroy_data = true;

    // every iteration --
    // 1. insert new data in the DB
    // 2. backup the DB
    // 3. destroy the db
    // 4. restore the db, check everything is still there
    for (int i = 0; i < 5; ++i) {
      // in last iteration, put smaller amount of data,
      int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
      // ---- insert new data and back up ----
      OpenDBAndBackupEngine(destroy_data);
      destroy_data = false;
      FillDB(db_.get(), keys_iteration * i, fill_up_to);
      ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
      CloseDBAndBackupEngine();
714
      DestroyDB(dbname_, options_);
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741

      // ---- make sure it's empty ----
      DB* db = OpenDB();
      AssertEmpty(db, 0, fill_up_to);
      delete db;

      // ---- restore the DB ----
      OpenBackupEngine();
      if (i >= 3) {  // test purge old backups
        // when i == 4, purge to only 1 backup
        // when i == 3, purge to 2 backups
        ASSERT_OK(backup_engine_->PurgeOldBackups(5 - i));
      }
      // ---- make sure the data is there ---
      AssertBackupConsistency(0, 0, fill_up_to, max_key);
      CloseBackupEngine();
    }
  }
}

// open DB, write, backup, write, backup, close, restore
TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
  // has to be a big number, so that it triggers the memtable flush
  const int keys_iteration = 5000;
  const int max_key = keys_iteration * 4 + 10;
  Random rnd(7);
  // delete old data
742
  DestroyDB(dbname_, options_);
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760

  OpenDBAndBackupEngine(true);
  // write some data, backup, repeat
  for (int i = 0; i < 5; ++i) {
    if (i == 4) {
      // delete backup number 2, online delete!
      ASSERT_OK(backup_engine_->DeleteBackup(2));
    }
    // in last iteration, put smaller amount of data,
    // so that backups can share sst files
    int fill_up_to = std::min(keys_iteration * (i + 1), max_key);
    FillDB(db_.get(), keys_iteration * i, fill_up_to);
    // we should get consistent results with flush_before_backup
    // set to both true and false
    ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
  }
  // close and destroy
  CloseDBAndBackupEngine();
761
  DestroyDB(dbname_, options_);
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799

  // ---- make sure it's empty ----
  DB* db = OpenDB();
  AssertEmpty(db, 0, max_key);
  delete db;

  // ---- restore every backup and verify all the data is there ----
  OpenBackupEngine();
  for (int i = 1; i <= 5; ++i) {
    if (i == 2) {
      // we deleted backup 2
      Status s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_);
      ASSERT_TRUE(!s.ok());
    } else {
      int fill_up_to = std::min(keys_iteration * i, max_key);
      AssertBackupConsistency(i, 0, fill_up_to, max_key);
    }
  }

  // delete some backups -- this should leave only backups 3 and 5 alive
  ASSERT_OK(backup_engine_->DeleteBackup(4));
  ASSERT_OK(backup_engine_->PurgeOldBackups(2));

  std::vector<BackupInfo> backup_info;
  backup_engine_->GetBackupInfo(&backup_info);
  ASSERT_EQ(2UL, backup_info.size());

  // check backup 3
  AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key);
  // check backup 5
  AssertBackupConsistency(5, 0, max_key);

  CloseBackupEngine();
}

INSTANTIATE_TEST_CASE_P(BackupableDBTestWithParam, BackupableDBTestWithParam,
                        ::testing::Bool());

Igor Canadi's avatar
Igor Canadi committed
800
// this will make sure that backup does not copy the same file twice
Igor Sugak's avatar
Igor Sugak committed
801
TEST_F(BackupableDBTest, NoDoubleCopy) {
802
  OpenDBAndBackupEngine(true, true);
Igor Canadi's avatar
Igor Canadi committed
803

Andrew Kryczka's avatar
Andrew Kryczka committed
804
  // should write 5 DB files + one meta file
Igor Canadi's avatar
Igor Canadi committed
805
  test_backup_env_->SetLimitWrittenFiles(7);
Lei Jin's avatar
Lei Jin committed
806
  test_backup_env_->ClearWrittenFiles();
Igor Canadi's avatar
Igor Canadi committed
807
  test_db_env_->SetLimitWrittenFiles(0);
Wanning Jiang's avatar
Wanning Jiang committed
808
809
  dummy_db_->live_files_ = {"/00010.sst", "/00011.sst", "/CURRENT",
                            "/MANIFEST-01"};
Igor Canadi's avatar
Igor Canadi committed
810
  dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
Islam AbdelRahman's avatar
Islam AbdelRahman committed
811
  test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
812
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
Lei Jin's avatar
Lei Jin committed
813
  std::vector<std::string> should_have_written = {
814
815
      "/shared/.00010.sst.tmp", "/shared/.00011.sst.tmp", "/private/1/CURRENT",
      "/private/1/MANIFEST-01", "/private/1/00011.log",   "/meta/.1.tmp"};
816
  AppendPath(backupdir_, should_have_written);
Lei Jin's avatar
Lei Jin committed
817
  test_backup_env_->AssertWrittenFiles(should_have_written);
Igor Canadi's avatar
Igor Canadi committed
818

Andrew Kryczka's avatar
Andrew Kryczka committed
819
  // should write 4 new DB files + one meta file
Igor Canadi's avatar
Igor Canadi committed
820
821
  // should not write/copy 00010.sst, since it's already there!
  test_backup_env_->SetLimitWrittenFiles(6);
Lei Jin's avatar
Lei Jin committed
822
  test_backup_env_->ClearWrittenFiles();
Wanning Jiang's avatar
Wanning Jiang committed
823
824
825

  dummy_db_->live_files_ = {"/00010.sst", "/00015.sst", "/CURRENT",
                            "/MANIFEST-01"};
Igor Canadi's avatar
Igor Canadi committed
826
  dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
Islam AbdelRahman's avatar
Islam AbdelRahman committed
827
  test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
828
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
Igor Canadi's avatar
Igor Canadi committed
829
  // should not open 00010.sst - it's already there
Wanning Jiang's avatar
Wanning Jiang committed
830

831
832
833
  should_have_written = {"/shared/.00015.sst.tmp", "/private/2/CURRENT",
                         "/private/2/MANIFEST-01", "/private/2/00011.log",
                         "/meta/.2.tmp"};
834
  AppendPath(backupdir_, should_have_written);
Lei Jin's avatar
Lei Jin committed
835
  test_backup_env_->AssertWrittenFiles(should_have_written);
Igor Canadi's avatar
Igor Canadi committed
836

837
  ASSERT_OK(backup_engine_->DeleteBackup(1));
agiardullo's avatar
agiardullo committed
838
839
  ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst"));

Igor Canadi's avatar
Igor Canadi committed
840
  // 00011.sst was only in backup 1, should be deleted
agiardullo's avatar
agiardullo committed
841
842
843
  ASSERT_EQ(Status::NotFound(),
            test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst"));
  ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst"));
Igor Canadi's avatar
Igor Canadi committed
844
845

  // MANIFEST file size should be only 100
Siying Dong's avatar
Siying Dong committed
846
  uint64_t size = 0;
Igor Canadi's avatar
Igor Canadi committed
847
  test_backup_env_->GetFileSize(backupdir_ + "/private/2/MANIFEST-01", &size);
848
  ASSERT_EQ(100UL, size);
Igor Canadi's avatar
Igor Canadi committed
849
  test_backup_env_->GetFileSize(backupdir_ + "/shared/00015.sst", &size);
850
  ASSERT_EQ(200UL, size);
851

852
  CloseDBAndBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
853
854
855
856
857
}

// test various kind of corruptions that may happen:
// 1. Not able to write a file for backup - that backup should fail,
//      everything else should work
Andrew Kryczka's avatar
Andrew Kryczka committed
858
// 2. Corrupted backup meta file or missing backuped file - we should
Igor Canadi's avatar
Igor Canadi committed
859
860
//      not be able to open that backup, but all other backups should be
//      fine
Andrew Kryczka's avatar
Andrew Kryczka committed
861
// 3. Corrupted checksum value - if the checksum is not a valid uint32_t,
Lei Jin's avatar
Lei Jin committed
862
//      db open should fail, otherwise, it aborts during the restore process.
Igor Sugak's avatar
Igor Sugak committed
863
TEST_F(BackupableDBTest, CorruptionsTest) {
864
  const int keys_iteration = 5000;
Igor Canadi's avatar
Igor Canadi committed
865
866
867
  Random rnd(6);
  Status s;

868
  OpenDBAndBackupEngine(true);
Igor Canadi's avatar
Igor Canadi committed
869
870
871
  // create five backups
  for (int i = 0; i < 5; ++i) {
    FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1));
872
    ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
Igor Canadi's avatar
Igor Canadi committed
873
874
875
876
877
878
879
  }

  // ---------- case 1. - fail a write -----------
  // try creating backup 6, but fail a write
  FillDB(db_.get(), keys_iteration * 5, keys_iteration * 6);
  test_backup_env_->SetLimitWrittenFiles(2);
  // should fail
880
  s = backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2));
Igor Canadi's avatar
Igor Canadi committed
881
882
883
  ASSERT_TRUE(!s.ok());
  test_backup_env_->SetLimitWrittenFiles(1000000);
  // latest backup should have all the keys
884
  CloseDBAndBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
885
886
  AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6);

Andrew Kryczka's avatar
Andrew Kryczka committed
887
  // --------- case 2. corrupted backup meta or missing backuped file ----
Igor Canadi's avatar
Igor Canadi committed
888
889
890
  ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3));
  // since 5 meta is now corrupted, latest backup should be 4
  AssertBackupConsistency(0, 0, keys_iteration * 4, keys_iteration * 5);
891
892
  OpenBackupEngine();
  s = backup_engine_->RestoreDBFromBackup(5, dbname_, dbname_);
Igor Canadi's avatar
Igor Canadi committed
893
  ASSERT_TRUE(!s.ok());
894
  CloseBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
895
896
897
  ASSERT_OK(file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/4"));
  // 4 is corrupted, 3 is the latest backup now
  AssertBackupConsistency(0, 0, keys_iteration * 3, keys_iteration * 5);
898
899
900
  OpenBackupEngine();
  s = backup_engine_->RestoreDBFromBackup(4, dbname_, dbname_);
  CloseBackupEngine();
Igor Canadi's avatar
Igor Canadi committed
901
902
  ASSERT_TRUE(!s.ok());

Andrew Kryczka's avatar
Andrew Kryczka committed
903
  // --------- case 3. corrupted checksum value ----
Lei Jin's avatar
Lei Jin committed
904
905
906
907
908
909
910
  ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/3", false));
  // checksum of backup 3 is an invalid value, this can be detected at
  // db open time, and it reverts to the previous backup automatically
  AssertBackupConsistency(0, 0, keys_iteration * 2, keys_iteration * 5);
  // checksum of the backup 2 appears to be valid, this can cause checksum
  // mismatch and abort restore process
  ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/2", true));
agiardullo's avatar
agiardullo committed
911
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
912
  OpenBackupEngine();
agiardullo's avatar
agiardullo committed
913
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
914
  s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_);
Lei Jin's avatar
Lei Jin committed
915
  ASSERT_TRUE(!s.ok());
Hasnain Lakhani's avatar
Hasnain Lakhani committed
916
917

  // make sure that no corrupt backups have actually been deleted!
agiardullo's avatar
agiardullo committed
918
919
920
921
922
923
924
925
926
927
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/1"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/3"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/4"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/5"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/1"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/2"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/3"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/4"));
  ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/5"));
Hasnain Lakhani's avatar
Hasnain Lakhani committed
928
929

  // delete the corrupt backups and then make sure they're actually deleted
930
931
932
933
934
  ASSERT_OK(backup_engine_->DeleteBackup(5));
  ASSERT_OK(backup_engine_->DeleteBackup(4));
  ASSERT_OK(backup_engine_->DeleteBackup(3));
  ASSERT_OK(backup_engine_->DeleteBackup(2));
  (void)backup_engine_->GarbageCollect();
agiardullo's avatar
agiardullo committed
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/meta/5"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/private/5"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/meta/4"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/private/4"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/meta/3"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/private/3"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/meta/2"));
  ASSERT_EQ(Status::NotFound(),
            file_manager_->FileExists(backupdir_ + "/private/2"));
Hasnain Lakhani's avatar
Hasnain Lakhani committed
951

952
  CloseBackupEngine();
Hasnain Lakhani's avatar
Hasnain Lakhani committed
953
954
955
  AssertBackupConsistency(0, 0, keys_iteration * 1, keys_iteration * 5);

  // new backup should be 2!
956
  OpenDBAndBackupEngine();
Hasnain Lakhani's avatar
Hasnain Lakhani committed
957
  FillDB(db_.get(), keys_iteration * 1, keys_iteration * 2);
958
959
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
  CloseDBAndBackupEngine();
Hasnain Lakhani's avatar
Hasnain Lakhani committed
960
  AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 5);
Igor Canadi's avatar
Igor Canadi committed
961
962
}

963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
TEST_F(BackupableDBTest, InterruptCreationTest) {
  // Interrupt backup creation by failing new writes and failing cleanup of the
  // partial state. Then verify a subsequent backup can still succeed.
  const int keys_iteration = 5000;
  Random rnd(6);

  OpenDBAndBackupEngine(true /* destroy_old_data */);
  FillDB(db_.get(), 0, keys_iteration);
  test_backup_env_->SetLimitWrittenFiles(2);
  test_backup_env_->SetDeleteFileFailure(true);
  // should fail creation
  ASSERT_FALSE(
      backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)).ok());
  CloseDBAndBackupEngine();
  // should also fail cleanup so the tmp directory stays behind
978
  ASSERT_OK(backup_chroot_env_->FileExists(backupdir_ + "/private/1/"));
979
980
981
982
983
984
985
986
987
988

  OpenDBAndBackupEngine(false /* destroy_old_data */);
  test_backup_env_->SetLimitWrittenFiles(1000000);
  test_backup_env_->SetDeleteFileFailure(false);
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2)));
  // latest backup should have all the keys
  CloseDBAndBackupEngine();
  AssertBackupConsistency(0, 0, keys_iteration);
}

Wanning Jiang's avatar
Wanning Jiang committed
989
990
991
992
993
994
995
996
997
998
999
1000
inline std::string OptionsPath(std::string ret, int backupID) {
  ret += "/private/";
  ret += std::to_string(backupID);
  ret += "/";
  return ret;
}

// Backup the LATEST options file to
// "<backup_dir>/private/<backup_id>/OPTIONS<number>"

TEST_F(BackupableDBTest, BackupOptions) {
  OpenDBAndBackupEngine(true);
For faster browsing, not all history is shown. View entire blame