column_family_test.cc 104 KB
Newer Older
1
//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
Siying Dong's avatar
Siying Dong committed
2
3
4
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
5
6
7
8
9
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

Igor Canadi's avatar
Igor Canadi committed
10
11
12
#include <algorithm>
#include <vector>
#include <string>
13
#include <thread>
Igor Canadi's avatar
Igor Canadi committed
14

15
#include "db/db_impl.h"
16
#include "db/db_test_util.h"
17
#include "options/options_parser.h"
Dmitri Smirnov's avatar
Dmitri Smirnov committed
18
#include "port/port.h"
19
#include "rocksdb/db.h"
20
21
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
22
#include "util/coding.h"
23
#include "util/fault_injection_test_env.h"
24
#include "util/string_util.h"
25
#include "util/sync_point.h"
26
#include "util/testharness.h"
27
#include "util/testutil.h"
28
#include "utilities/merge_operators.h"
29
30
31

namespace rocksdb {

32
33
static const int kValueSize = 1000;

34
35
36
37
38
39
40
41
namespace {
std::string RandomString(Random* rnd, int len) {
  std::string r;
  test::RandomString(rnd, len, &r);
  return r;
}
}  // anonymous namespace

Igor Canadi's avatar
Igor Canadi committed
42
43
44
45
46
47
48
49
50
// counts how many operations were performed
class EnvCounter : public EnvWrapper {
 public:
  explicit EnvCounter(Env* base)
      : EnvWrapper(base), num_new_writable_file_(0) {}
  int GetNumberOfNewWritableFileCalls() {
    return num_new_writable_file_;
  }
  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
Igor Sugak's avatar
Igor Sugak committed
51
                         const EnvOptions& soptions) override {
Igor Canadi's avatar
Igor Canadi committed
52
53
54
55
56
    ++num_new_writable_file_;
    return EnvWrapper::NewWritableFile(f, r, soptions);
  }

 private:
57
  std::atomic<int> num_new_writable_file_;
Igor Canadi's avatar
Igor Canadi committed
58
59
};

Igor Sugak's avatar
Igor Sugak committed
60
class ColumnFamilyTest : public testing::Test {
61
 public:
62
  ColumnFamilyTest() : rnd_(139) {
Igor Canadi's avatar
Igor Canadi committed
63
    env_ = new EnvCounter(Env::Default());
64
65
    dbname_ = test::TmpDir() + "/column_family_test";
    db_options_.create_if_missing = true;
66
    db_options_.fail_if_options_file_error = true;
Igor Canadi's avatar
Igor Canadi committed
67
    db_options_.env = env_;
68
    DestroyDB(dbname_, Options(db_options_, column_family_options_));
69
70
  }

71
  ~ColumnFamilyTest() {
72
    Close();
73
    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
74
    Destroy();
75
76
77
    delete env_;
  }

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  // Return the value to associate with the specified key
  Slice Value(int k, std::string* storage) {
    if (k == 0) {
      // Ugh.  Random seed of 0 used to produce no entropy.  This code
      // preserves the implementation that was in place when all of the
      // magic values in this file were picked.
      *storage = std::string(kValueSize, ' ');
      return Slice(*storage);
    } else {
      Random r(k);
      return test::RandomString(&r, kValueSize, storage);
    }
  }

  void Build(int base, int n, int flush_every = 0) {
    std::string key_space, value_space;
    WriteBatch batch;

    for (int i = 0; i < n; i++) {
      if (flush_every != 0 && i != 0 && i % flush_every == 0) {
        DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
        dbi->TEST_FlushMemTable();
      }

      int keyi = base + i;
      Slice key(DBTestBase::Key(keyi));

      batch.Clear();
      batch.Put(handles_[0], key, Value(keyi, &value_space));
      batch.Put(handles_[1], key, Value(keyi, &value_space));
      batch.Put(handles_[2], key, Value(keyi, &value_space));
      ASSERT_OK(db_->Write(WriteOptions(), &batch));
    }
  }

  void CheckMissed() {
    uint64_t next_expected = 0;
    uint64_t missed = 0;
    int bad_keys = 0;
    int bad_values = 0;
    int correct = 0;
    std::string value_space;
    for (int cf = 0; cf < 3; cf++) {
      next_expected = 0;
      Iterator* iter = db_->NewIterator(ReadOptions(false, true), handles_[cf]);
      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
        uint64_t key;
        Slice in(iter->key());
        in.remove_prefix(3);
        if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
            key < next_expected) {
          bad_keys++;
          continue;
        }
        missed += (key - next_expected);
        next_expected = key + 1;
        if (iter->value() != Value(static_cast<int>(key), &value_space)) {
          bad_values++;
        } else {
          correct++;
        }
      }
      delete iter;
    }

    ASSERT_EQ(0, bad_keys);
    ASSERT_EQ(0, bad_values);
    ASSERT_EQ(0, missed);
    (void)correct;
  }

149
  void Close() {
150
    for (auto h : handles_) {
151
      if (h) {
152
        db_->DestroyColumnFamilyHandle(h);
153
      }
154
155
    }
    handles_.clear();
Igor Canadi's avatar
Igor Canadi committed
156
    names_.clear();
157
158
159
160
    delete db_;
    db_ = nullptr;
  }

Igor Canadi's avatar
Igor Canadi committed
161
162
  Status TryOpen(std::vector<std::string> cf,
                 std::vector<ColumnFamilyOptions> options = {}) {
Igor Canadi's avatar
Igor Canadi committed
163
    std::vector<ColumnFamilyDescriptor> column_families;
Igor Canadi's avatar
Igor Canadi committed
164
165
166
167
168
    names_.clear();
    for (size_t i = 0; i < cf.size(); ++i) {
      column_families.push_back(ColumnFamilyDescriptor(
          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
      names_.push_back(cf[i]);
169
    }
170
    return DB::Open(db_options_, dbname_, column_families, &handles_, &db_);
171
172
  }

173
174
175
176
177
178
179
180
181
182
183
184
185
  Status OpenReadOnly(std::vector<std::string> cf,
                         std::vector<ColumnFamilyOptions> options = {}) {
    std::vector<ColumnFamilyDescriptor> column_families;
    names_.clear();
    for (size_t i = 0; i < cf.size(); ++i) {
      column_families.push_back(ColumnFamilyDescriptor(
          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
      names_.push_back(cf[i]);
    }
    return DB::OpenForReadOnly(db_options_, dbname_, column_families, &handles_,
                               &db_);
  }

186
#ifndef ROCKSDB_LITE  // ReadOnlyDB is not supported
187
188
189
190
  void AssertOpenReadOnly(std::vector<std::string> cf,
                    std::vector<ColumnFamilyOptions> options = {}) {
    ASSERT_OK(OpenReadOnly(cf, options));
  }
191
#endif  // !ROCKSDB_LITE
192
193


Igor Canadi's avatar
Igor Canadi committed
194
195
196
197
198
199
200
201
202
  void Open(std::vector<std::string> cf,
            std::vector<ColumnFamilyOptions> options = {}) {
    ASSERT_OK(TryOpen(cf, options));
  }

  void Open() {
    Open({"default"});
  }

203
204
  DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }

Igor Canadi's avatar
Igor Canadi committed
205
206
  int GetProperty(int cf, std::string property) {
    std::string value;
207
    EXPECT_TRUE(dbfull()->GetProperty(handles_[cf], property, &value));
sdong's avatar
sdong committed
208
#ifndef CYGWIN
Igor Canadi's avatar
Igor Canadi committed
209
    return std::stoi(value);
sdong's avatar
sdong committed
210
#else
sdong's avatar
sdong committed
211
    return std::strtol(value.c_str(), 0 /* off */, 10 /* base */);
sdong's avatar
sdong committed
212
#endif
Igor Canadi's avatar
Igor Canadi committed
213
214
  }

215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
  bool IsDbWriteStopped() {
#ifndef ROCKSDB_LITE
    uint64_t v;
    EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.is-write-stopped", &v));
    return (v == 1);
#else
    return dbfull()->TEST_write_controler().IsStopped();
#endif  // !ROCKSDB_LITE
  }

  uint64_t GetDbDelayedWriteRate() {
#ifndef ROCKSDB_LITE
    uint64_t v;
    EXPECT_TRUE(
        dbfull()->GetIntProperty("rocksdb.actual-delayed-write-rate", &v));
    return v;
#else
    if (!dbfull()->TEST_write_controler().NeedsDelay()) {
      return 0;
    }
    return dbfull()->TEST_write_controler().delayed_write_rate();
#endif  // !ROCKSDB_LITE
  }

239
  void Destroy() {
240
    Close();
241
242
243
    ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
  }

Igor Canadi's avatar
Igor Canadi committed
244
245
246
  void CreateColumnFamilies(
      const std::vector<std::string>& cfs,
      const std::vector<ColumnFamilyOptions> options = {}) {
247
    int cfi = static_cast<int>(handles_.size());
248
    handles_.resize(cfi + cfs.size());
Igor Canadi's avatar
Igor Canadi committed
249
250
    names_.resize(cfi + cfs.size());
    for (size_t i = 0; i < cfs.size(); ++i) {
251
252
253
254
      const auto& current_cf_opt =
          options.size() == 0 ? column_family_options_ : options[i];
      ASSERT_OK(
          db_->CreateColumnFamily(current_cf_opt, cfs[i], &handles_[cfi]));
Igor Canadi's avatar
Igor Canadi committed
255
      names_[cfi] = cfs[i];
256
257
258
259
260
261
262

#ifndef ROCKSDB_LITE  // RocksDBLite does not support GetDescriptor
      // Verify the CF options of the returned CF handle.
      ColumnFamilyDescriptor desc;
      ASSERT_OK(handles_[cfi]->GetDescriptor(&desc));
      RocksDBOptionsParser::VerifyCFOptions(desc.options, current_cf_opt);
#endif  // !ROCKSDB_LITE
Igor Canadi's avatar
Igor Canadi committed
263
      cfi++;
264
265
266
    }
  }

Igor Canadi's avatar
Igor Canadi committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
  void Reopen(const std::vector<ColumnFamilyOptions> options = {}) {
    std::vector<std::string> names;
    for (auto name : names_) {
      if (name != "") {
        names.push_back(name);
      }
    }
    Close();
    assert(options.size() == 0 || names.size() == options.size());
    Open(names, options);
  }

  void CreateColumnFamiliesAndReopen(const std::vector<std::string>& cfs) {
    CreateColumnFamilies(cfs);
    Reopen();
  }

Igor Canadi's avatar
Igor Canadi committed
284
  void DropColumnFamilies(const std::vector<int>& cfs) {
285
286
    for (auto cf : cfs) {
      ASSERT_OK(db_->DropColumnFamily(handles_[cf]));
287
      db_->DestroyColumnFamilyHandle(handles_[cf]);
288
      handles_[cf] = nullptr;
Igor Canadi's avatar
Igor Canadi committed
289
      names_[cf] = "";
290
291
292
    }
  }

293
  void PutRandomData(int cf, int num, int key_value_size, bool save = false) {
Igor Canadi's avatar
Igor Canadi committed
294
295
    for (int i = 0; i < num; ++i) {
      // 10 bytes for key, rest is value
296
297
298
299
300
301
302
303
      if (!save) {
        ASSERT_OK(Put(cf, test::RandomKey(&rnd_, 11),
                      RandomString(&rnd_, key_value_size - 10)));
      } else {
        std::string key = test::RandomKey(&rnd_, 11);
        keys_.insert(key);
        ASSERT_OK(Put(cf, key, RandomString(&rnd_, key_value_size - 10)));
      }
304
    }
305
    db_->FlushWAL(false);
306
307
  }

308
#ifndef ROCKSDB_LITE  // TEST functions in DB are not supported in lite
309
310
311
312
  void WaitForFlush(int cf) {
    ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
  }

313
314
315
316
317
318
319
320
321
322
323
  void WaitForCompaction() {
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
  }

  uint64_t MaxTotalInMemoryState() {
    return dbfull()->TEST_MaxTotalInMemoryState();
  }

  void AssertMaxTotalInMemoryState(uint64_t value) {
    ASSERT_EQ(value, MaxTotalInMemoryState());
  }
324
#endif  // !ROCKSDB_LITE
325

Igor Canadi's avatar
Igor Canadi committed
326
  Status Put(int cf, const std::string& key, const std::string& value) {
327
328
    return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(value));
  }
Igor Canadi's avatar
Igor Canadi committed
329
  Status Merge(int cf, const std::string& key, const std::string& value) {
330
331
    return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
  }
332
333
334
  Status Flush(int cf) {
    return db_->Flush(FlushOptions(), handles_[cf]);
  }
335

Igor Canadi's avatar
Igor Canadi committed
336
  std::string Get(int cf, const std::string& key) {
337
338
    ReadOptions options;
    options.verify_checksums = true;
Igor Canadi's avatar
Igor Canadi committed
339
    std::string result;
340
341
342
343
344
345
346
347
348
    Status s = db_->Get(options, handles_[cf], Slice(key), &result);
    if (s.IsNotFound()) {
      result = "NOT_FOUND";
    } else if (!s.ok()) {
      result = s.ToString();
    }
    return result;
  }

349
  void CompactAll(int cf) {
350
351
    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[cf], nullptr,
                                nullptr));
352
353
  }

Igor Canadi's avatar
Igor Canadi committed
354
  void Compact(int cf, const Slice& start, const Slice& limit) {
355
356
    ASSERT_OK(
        db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit));
Igor Canadi's avatar
Igor Canadi committed
357
358
  }

359
360
  int NumTableFilesAtLevel(int level, int cf) {
    return GetProperty(cf,
361
                       "rocksdb.num-files-at-level" + ToString(level));
Igor Canadi's avatar
Igor Canadi committed
362
363
  }

364
#ifndef ROCKSDB_LITE
Igor Canadi's avatar
Igor Canadi committed
365
  // Return spread of files per level
Igor Canadi's avatar
Igor Canadi committed
366
367
  std::string FilesPerLevel(int cf) {
    std::string result;
Igor Canadi's avatar
Igor Canadi committed
368
    int last_non_zero_offset = 0;
369
370
    for (int level = 0; level < dbfull()->NumberLevels(handles_[cf]); level++) {
      int f = NumTableFilesAtLevel(level, cf);
Igor Canadi's avatar
Igor Canadi committed
371
372
373
374
      char buf[100];
      snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
      result += buf;
      if (f > 0) {
375
        last_non_zero_offset = static_cast<int>(result.size());
Igor Canadi's avatar
Igor Canadi committed
376
377
378
379
380
      }
    }
    result.resize(last_non_zero_offset);
    return result;
  }
381
#endif
Igor Canadi's avatar
Igor Canadi committed
382

383
384
385
  void AssertFilesPerLevel(const std::string& value, int cf) {
#ifndef ROCKSDB_LITE
    ASSERT_EQ(value, FilesPerLevel(cf));
386
387
388
#else
    (void) value;
    (void) cf;
389
390
391
392
#endif
  }

#ifndef ROCKSDB_LITE  // GetLiveFilesMetaData is not supported
Igor Canadi's avatar
Igor Canadi committed
393
  int CountLiveFiles() {
394
395
396
397
    std::vector<LiveFileMetaData> metadata;
    db_->GetLiveFilesMetaData(&metadata);
    return static_cast<int>(metadata.size());
  }
398
399
400
401
402
#endif  // !ROCKSDB_LITE

  void AssertCountLiveFiles(int expected_value) {
#ifndef ROCKSDB_LITE
    ASSERT_EQ(expected_value, CountLiveFiles());
403
404
#else
    (void) expected_value;
405
406
#endif
  }
407

Igor Canadi's avatar
Igor Canadi committed
408
409
  // Do n memtable flushes, each of which produces an sstable
  // covering the range [small,large].
Igor Canadi's avatar
Igor Canadi committed
410
411
  void MakeTables(int cf, int n, const std::string& small,
                  const std::string& large) {
Igor Canadi's avatar
Igor Canadi committed
412
413
414
415
416
417
418
    for (int i = 0; i < n; i++) {
      ASSERT_OK(Put(cf, small, "begin"));
      ASSERT_OK(Put(cf, large, "end"));
      ASSERT_OK(db_->Flush(FlushOptions(), handles_[cf]));
    }
  }

419
#ifndef ROCKSDB_LITE  // GetSortedWalFiles is not supported
420
  int CountLiveLogFiles() {
421
422
    int micros_wait_for_log_deletion = 20000;
    env_->SleepForMicroseconds(micros_wait_for_log_deletion);
423
424
    int ret = 0;
    VectorLogPtr wal_files;
425
426
    Status s;
    // GetSortedWalFiles is a flakey function -- it gets all the wal_dir
clark.kang's avatar
clark.kang committed
427
    // children files and then later checks for their existence. if some of the
428
429
430
431
432
433
434
435
436
437
438
    // log files doesn't exist anymore, it reports an error. it does all of this
    // without DB mutex held, so if a background process deletes the log file
    // while the function is being executed, it returns an error. We retry the
    // function 10 times to avoid the error failing the test
    for (int retries = 0; retries < 10; ++retries) {
      wal_files.clear();
      s = db_->GetSortedWalFiles(wal_files);
      if (s.ok()) {
        break;
      }
    }
439
    EXPECT_OK(s);
440
441
442
443
444
445
    for (const auto& wal : wal_files) {
      if (wal->Type() == kAliveLogFile) {
        ++ret;
      }
    }
    return ret;
446
447
448
449
450
451
452
    return 0;
  }
#endif  // !ROCKSDB_LITE

  void AssertCountLiveLogFiles(int value) {
#ifndef ROCKSDB_LITE  // GetSortedWalFiles is not supported
    ASSERT_EQ(value, CountLiveLogFiles());
453
454
#else
    (void) value;
455
#endif  // !ROCKSDB_LITE
456
457
  }

Igor Canadi's avatar
Igor Canadi committed
458
459
460
  void AssertNumberOfImmutableMemtables(std::vector<int> num_per_cf) {
    assert(num_per_cf.size() == handles_.size());

461
#ifndef ROCKSDB_LITE  // GetProperty is not supported in lite
Igor Canadi's avatar
Igor Canadi committed
462
    for (size_t i = 0; i < num_per_cf.size(); ++i) {
463
464
      ASSERT_EQ(num_per_cf[i], GetProperty(static_cast<int>(i),
                                           "rocksdb.num-immutable-mem-table"));
Igor Canadi's avatar
Igor Canadi committed
465
    }
466
#endif  // !ROCKSDB_LITE
Igor Canadi's avatar
Igor Canadi committed
467
468
  }

Igor Canadi's avatar
Igor Canadi committed
469
  void CopyFile(const std::string& source, const std::string& destination,
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
                uint64_t size = 0) {
    const EnvOptions soptions;
    unique_ptr<SequentialFile> srcfile;
    ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions));
    unique_ptr<WritableFile> destfile;
    ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions));

    if (size == 0) {
      // default argument means copy everything
      ASSERT_OK(env_->GetFileSize(source, &size));
    }

    char buffer[4096];
    Slice slice;
    while (size > 0) {
Igor Canadi's avatar
Igor Canadi committed
485
      uint64_t one = std::min(uint64_t(sizeof(buffer)), size);
486
487
488
489
490
491
492
      ASSERT_OK(srcfile->Read(one, &slice, buffer));
      ASSERT_OK(destfile->Append(slice));
      size -= slice.size();
    }
    ASSERT_OK(destfile->Close());
  }

Igor Canadi's avatar
Igor Canadi committed
493
  std::vector<ColumnFamilyHandle*> handles_;
Igor Canadi's avatar
Igor Canadi committed
494
  std::vector<std::string> names_;
495
  std::set<std::string> keys_;
496
497
  ColumnFamilyOptions column_family_options_;
  DBOptions db_options_;
Igor Canadi's avatar
Igor Canadi committed
498
  std::string dbname_;
Igor Canadi's avatar
Igor Canadi committed
499
  DB* db_ = nullptr;
Igor Canadi's avatar
Igor Canadi committed
500
  EnvCounter* env_;
501
  Random rnd_;
502
503
};

Igor Sugak's avatar
Igor Sugak committed
504
TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
505
506
507
508
  for (int iter = 0; iter < 3; ++iter) {
    Open();
    CreateColumnFamilies({"one", "two", "three"});
    for (size_t i = 0; i < handles_.size(); ++i) {
509
510
      auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[i]);
      ASSERT_EQ(i, cfh->GetID());
511
512
513
514
515
516
517
518
519
520
521
522
523
    }
    if (iter == 1) {
      Reopen();
    }
    DropColumnFamilies({3});
    Reopen();
    if (iter == 2) {
      // this tests if max_column_family is correctly persisted with
      // WriteSnapshot()
      Reopen();
    }
    CreateColumnFamilies({"three2"});
    // ID 3 that was used for dropped column family "three" should not be reused
524
    auto cfh3 = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[3]);
Igor Canadi's avatar
Igor Canadi committed
525
    ASSERT_EQ(4U, cfh3->GetID());
526
527
528
529
530
    Close();
    Destroy();
  }
}

Yi Wu's avatar
Yi Wu committed
531
#ifndef ROCKSDB_LITE
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
  Open();

  rocksdb::SyncPoint::GetInstance()->LoadDependency(
      {{"DBImpl::WriteOptionsFile:1",
        "ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1"},
       {"ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2",
        "DBImpl::WriteOptionsFile:2"}});
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

  rocksdb::port::Thread thread([&] { CreateColumnFamilies({"one"}); });

  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1");
  uint64_t pv;
  db_->GetAggregatedIntProperty(DB::Properties::kEstimateTableReadersMem, &pv);
  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2");

  thread.join();

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
}
Yi Wu's avatar
Yi Wu committed
553
#endif  // !ROCKSDB_LITE
554

sdong's avatar
sdong committed
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                  public testing::WithParamInterface<bool> {
 public:
  FlushEmptyCFTestWithParam() { allow_2pc_ = GetParam(); }

  // Required if inheriting from testing::WithParamInterface<>
  static void SetUpTestCase() {}
  static void TearDownTestCase() {}

  bool allow_2pc_;
};

TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) {
  std::unique_ptr<FaultInjectionTestEnv> fault_env(
      new FaultInjectionTestEnv(env_));
  db_options_.env = fault_env.get();
  db_options_.allow_2pc = allow_2pc_;
  Open();
  CreateColumnFamilies({"one", "two"});
  // Generate log file A.
  ASSERT_OK(Put(1, "foo", "v1"));  // seqID 1

  Reopen();
  // Log file A is not dropped after reopening because default column family's
  // min log number is 0.
  // It flushes to SST file X
  ASSERT_OK(Put(1, "foo", "v1"));  // seqID 2
  ASSERT_OK(Put(1, "bar", "v2"));  // seqID 3
  // Current log file is file B now. While flushing, a new log file C is created
  // and is set to current. Boths' min log number is set to file C in memory, so
  // after flushing file B is deleted. At the same time, the min log number of
  // default CF is not written to manifest. Log file A still remains.
  // Flushed to SST file Y.
  Flush(1);
  Flush(0);
  ASSERT_OK(Put(1, "bar", "v3"));  // seqID 4
  ASSERT_OK(Put(1, "foo", "v4"));  // seqID 5
592
  db_->FlushWAL(false);
sdong's avatar
sdong committed
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654

  // Preserve file system state up to here to simulate a crash condition.
  fault_env->SetFilesystemActive(false);
  std::vector<std::string> names;
  for (auto name : names_) {
    if (name != "") {
      names.push_back(name);
    }
  }

  Close();
  fault_env->ResetState();

  // Before opening, there are four files:
  //   Log file A contains seqID 1
  //   Log file C contains seqID 4, 5
  //   SST file X contains seqID 1
  //   SST file Y contains seqID 2, 3
  // Min log number:
  //   default CF: 0
  //   CF one, two: C
  // When opening the DB, all the seqID should be preserved.
  Open(names, {});
  ASSERT_EQ("v4", Get(1, "foo"));
  ASSERT_EQ("v3", Get(1, "bar"));
  Close();

  db_options_.env = env_;
}

TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
  std::unique_ptr<FaultInjectionTestEnv> fault_env(
      new FaultInjectionTestEnv(env_));
  db_options_.env = fault_env.get();
  db_options_.allow_2pc = allow_2pc_;
  Open();
  CreateColumnFamilies({"one", "two"});
  // Generate log file A.
  ASSERT_OK(Put(1, "foo", "v1"));  // seqID 1

  Reopen();
  // Log file A is not dropped after reopening because default column family's
  // min log number is 0.
  // It flushes to SST file X
  ASSERT_OK(Put(1, "foo", "v1"));  // seqID 2
  ASSERT_OK(Put(1, "bar", "v2"));  // seqID 3
  // Current log file is file B now. While flushing, a new log file C is created
  // and is set to current. Both CFs' min log number is set to file C so after
  // flushing file B is deleted. Log file A still remains.
  // Flushed to SST file Y.
  Flush(1);
  ASSERT_OK(Put(0, "bar", "v2"));  // seqID 4
  ASSERT_OK(Put(2, "bar", "v2"));  // seqID 5
  ASSERT_OK(Put(1, "bar", "v3"));  // seqID 6
  // Flushing all column families. This forces all CFs' min log to current. This
  // is written to the manifest file. Log file C is cleared.
  Flush(0);
  Flush(1);
  Flush(2);
  // Write to log file D
  ASSERT_OK(Put(1, "bar", "v4"));  // seqID 7
  ASSERT_OK(Put(1, "bar", "v5"));  // seqID 8
655
  db_->FlushWAL(false);
sdong's avatar
sdong committed
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
  // Preserve file system state up to here to simulate a crash condition.
  fault_env->SetFilesystemActive(false);
  std::vector<std::string> names;
  for (auto name : names_) {
    if (name != "") {
      names.push_back(name);
    }
  }

  Close();
  fault_env->ResetState();
  // Before opening, there are two logfiles:
  //   Log file A contains seqID 1
  //   Log file D contains seqID 7, 8
  // Min log number:
  //   default CF: D
  //   CF one, two: D
  // When opening the DB, log file D should be replayed using the seqID
  // specified in the file.
  Open(names, {});
  ASSERT_EQ("v1", Get(1, "foo"));
  ASSERT_EQ("v5", Get(1, "bar"));
  Close();

  db_options_.env = env_;
}

INSTANTIATE_TEST_CASE_P(FlushEmptyCFTestWithParam, FlushEmptyCFTestWithParam,
                        ::testing::Bool());

Igor Sugak's avatar
Igor Sugak committed
686
TEST_F(ColumnFamilyTest, AddDrop) {
Igor Canadi's avatar
Igor Canadi committed
687
  Open();
688
  CreateColumnFamilies({"one", "two", "three"});
Igor Canadi's avatar
Igor Canadi committed
689
690
  ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
  ASSERT_EQ("NOT_FOUND", Get(2, "fodor"));
691
  DropColumnFamilies({2});
Igor Canadi's avatar
Igor Canadi committed
692
  ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
693
  CreateColumnFamilies({"four"});
Igor Canadi's avatar
Igor Canadi committed
694
695
696
697
  ASSERT_EQ("NOT_FOUND", Get(3, "fodor"));
  ASSERT_OK(Put(1, "fodor", "mirko"));
  ASSERT_EQ("mirko", Get(1, "fodor"));
  ASSERT_EQ("NOT_FOUND", Get(3, "fodor"));
698
  Close();
Igor Canadi's avatar
Igor Canadi committed
699
700
701
702
  ASSERT_TRUE(TryOpen({"default"}).IsInvalidArgument());
  Open({"default", "one", "three", "four"});
  DropColumnFamilies({1});
  Reopen();
703
704
  Close();

Igor Canadi's avatar
Igor Canadi committed
705
  std::vector<std::string> families;
Igor Canadi's avatar
Igor Canadi committed
706
  ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families));
707
  std::sort(families.begin(), families.end());
Igor Canadi's avatar
Igor Canadi committed
708
  ASSERT_TRUE(families ==
Igor Canadi's avatar
Igor Canadi committed
709
              std::vector<std::string>({"default", "four", "three"}));
710
711
}

Yi Wu's avatar
Yi Wu committed
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
TEST_F(ColumnFamilyTest, BulkAddDrop) {
  constexpr int kNumCF = 1000;
  ColumnFamilyOptions cf_options;
  WriteOptions write_options;
  Open();
  std::vector<std::string> cf_names;
  std::vector<ColumnFamilyHandle*> cf_handles;
  for (int i = 1; i <= kNumCF; i++) {
    cf_names.push_back("cf1-" + ToString(i));
  }
  ASSERT_OK(db_->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
  for (int i = 1; i <= kNumCF; i++) {
    ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar"));
  }
  ASSERT_OK(db_->DropColumnFamilies(cf_handles));
  std::vector<ColumnFamilyDescriptor> cf_descriptors;
Yi Wu's avatar
Yi Wu committed
728
729
730
  for (auto* handle : cf_handles) {
    delete handle;
  }
Yi Wu's avatar
Yi Wu committed
731
732
733
734
735
736
737
738
739
  cf_handles.clear();
  for (int i = 1; i <= kNumCF; i++) {
    cf_descriptors.emplace_back("cf2-" + ToString(i), ColumnFamilyOptions());
  }
  ASSERT_OK(db_->CreateColumnFamilies(cf_descriptors, &cf_handles));
  for (int i = 1; i <= kNumCF; i++) {
    ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar"));
  }
  ASSERT_OK(db_->DropColumnFamilies(cf_handles));
Yi Wu's avatar
Yi Wu committed
740
741
742
  for (auto* handle : cf_handles) {
    delete handle;
  }
Yi Wu's avatar
Yi Wu committed
743
744
745
746
747
748
749
  Close();
  std::vector<std::string> families;
  ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families));
  std::sort(families.begin(), families.end());
  ASSERT_TRUE(families == std::vector<std::string>({"default"}));
}

Igor Sugak's avatar
Igor Sugak committed
750
TEST_F(ColumnFamilyTest, DropTest) {
751
752
753
  // first iteration - dont reopen DB before dropping
  // second iteration - reopen DB before dropping
  for (int iter = 0; iter < 2; ++iter) {
Igor Canadi's avatar
Igor Canadi committed
754
755
    Open({"default"});
    CreateColumnFamiliesAndReopen({"pikachu"});
756
    for (int i = 0; i < 100; ++i) {
757
      ASSERT_OK(Put(1, ToString(i), "bar" + ToString(i)));
758
759
760
761
    }
    ASSERT_OK(Flush(1));

    if (iter == 1) {
Igor Canadi's avatar
Igor Canadi committed
762
      Reopen();
763
764
765
    }
    ASSERT_EQ("bar1", Get(1, "1"));

766
    AssertCountLiveFiles(1);
767
768
    DropColumnFamilies({1});
    // make sure that all files are deleted when we drop the column family
769
    AssertCountLiveFiles(0);
770
771
772
773
    Destroy();
  }
}

Igor Sugak's avatar
Igor Sugak committed
774
TEST_F(ColumnFamilyTest, WriteBatchFailure) {
775
  Open();
776
  CreateColumnFamiliesAndReopen({"one", "two"});
777
  WriteBatch batch;
Igor Canadi's avatar
Igor Canadi committed
778
  batch.Put(handles_[0], Slice("existing"), Slice("column-family"));
779
780
781
  batch.Put(handles_[1], Slice("non-existing"), Slice("column-family"));
  ASSERT_OK(db_->Write(WriteOptions(), &batch));
  DropColumnFamilies({1});
Igor Canadi's avatar
Igor Canadi committed
782
783
784
785
786
  WriteOptions woptions_ignore_missing_cf;
  woptions_ignore_missing_cf.ignore_missing_column_families = true;
  batch.Put(handles_[0], Slice("still here"), Slice("column-family"));
  ASSERT_OK(db_->Write(woptions_ignore_missing_cf, &batch));
  ASSERT_EQ("column-family", Get(0, "still here"));
787
788
789
790
791
  Status s = db_->Write(WriteOptions(), &batch);
  ASSERT_TRUE(s.IsInvalidArgument());
  Close();
}

Igor Sugak's avatar
Igor Sugak committed
792
TEST_F(ColumnFamilyTest, ReadWrite) {
Igor Canadi's avatar
Igor Canadi committed
793
794
  Open();
  CreateColumnFamiliesAndReopen({"one", "two"});
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
  ASSERT_OK(Put(0, "foo", "v1"));
  ASSERT_OK(Put(0, "bar", "v2"));
  ASSERT_OK(Put(1, "mirko", "v3"));
  ASSERT_OK(Put(0, "foo", "v2"));
  ASSERT_OK(Put(2, "fodor", "v5"));

  for (int iter = 0; iter <= 3; ++iter) {
    ASSERT_EQ("v2", Get(0, "foo"));
    ASSERT_EQ("v2", Get(0, "bar"));
    ASSERT_EQ("v3", Get(1, "mirko"));
    ASSERT_EQ("v5", Get(2, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(0, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(2, "foo"));
    if (iter <= 1) {
Igor Canadi's avatar
Igor Canadi committed
810
      Reopen();
811
812
813
814
815
    }
  }
  Close();
}

Igor Sugak's avatar
Igor Sugak committed
816
TEST_F(ColumnFamilyTest, IgnoreRecoveredLog) {
Igor Canadi's avatar
Igor Canadi committed
817
  std::string backup_logs = dbname_ + "/backup_logs";
818
819

  // delete old files in backup_logs directory
820
821
  ASSERT_OK(env_->CreateDirIfMissing(dbname_));
  ASSERT_OK(env_->CreateDirIfMissing(backup_logs));
Igor Canadi's avatar
Igor Canadi committed
822
  std::vector<std::string> old_files;
823
824
825
826
827
828
829
830
831
832
833
  env_->GetChildren(backup_logs, &old_files);
  for (auto& file : old_files) {
    if (file != "." && file != "..") {
      env_->DeleteFile(backup_logs + "/" + file);
    }
  }

  column_family_options_.merge_operator =
      MergeOperators::CreateUInt64AddOperator();
  db_options_.wal_dir = dbname_ + "/logs";
  Destroy();
Igor Canadi's avatar
Igor Canadi committed
834
  Open();
835
836
837
  CreateColumnFamilies({"cf1", "cf2"});

  // fill up the DB
Igor Canadi's avatar
Igor Canadi committed
838
  std::string one, two, three;
839
840
841
842
843
844
845
846
847
848
849
850
851
852
  PutFixed64(&one, 1);
  PutFixed64(&two, 2);
  PutFixed64(&three, 3);
  ASSERT_OK(Merge(0, "foo", one));
  ASSERT_OK(Merge(1, "mirko", one));
  ASSERT_OK(Merge(0, "foo", one));
  ASSERT_OK(Merge(2, "bla", one));
  ASSERT_OK(Merge(2, "fodor", one));
  ASSERT_OK(Merge(0, "bar", one));
  ASSERT_OK(Merge(2, "bla", one));
  ASSERT_OK(Merge(1, "mirko", two));
  ASSERT_OK(Merge(1, "franjo", one));

  // copy the logs to backup
Igor Canadi's avatar
Igor Canadi committed
853
  std::vector<std::string> logs;
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
  env_->GetChildren(db_options_.wal_dir, &logs);
  for (auto& log : logs) {
    if (log != ".." && log != ".") {
      CopyFile(db_options_.wal_dir + "/" + log, backup_logs + "/" + log);
    }
  }

  // recover the DB
  Close();

  // 1. check consistency
  // 2. copy the logs from backup back to WAL dir. if the recovery happens
  // again on the same log files, this should lead to incorrect results
  // due to applying merge operator twice
  // 3. check consistency
  for (int iter = 0; iter < 2; ++iter) {
    // assert consistency
Igor Canadi's avatar
Igor Canadi committed
871
    Open({"default", "cf1", "cf2"});
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
    ASSERT_EQ(two, Get(0, "foo"));
    ASSERT_EQ(one, Get(0, "bar"));
    ASSERT_EQ(three, Get(1, "mirko"));
    ASSERT_EQ(one, Get(1, "franjo"));
    ASSERT_EQ(one, Get(2, "fodor"));
    ASSERT_EQ(two, Get(2, "bla"));
    Close();

    if (iter == 0) {
      // copy the logs from backup back to wal dir
      for (auto& log : logs) {
        if (log != ".." && log != ".") {
          CopyFile(backup_logs + "/" + log, db_options_.wal_dir + "/" + log);
        }
      }
    }
  }
}

891
#ifndef ROCKSDB_LITE  // TEST functions used are not supported
Igor Sugak's avatar
Igor Sugak committed
892
TEST_F(ColumnFamilyTest, FlushTest) {
Igor Canadi's avatar
Igor Canadi committed
893
894
  Open();
  CreateColumnFamiliesAndReopen({"one", "two"});
895
896
897
898
899
  ASSERT_OK(Put(0, "foo", "v1"));
  ASSERT_OK(Put(0, "bar", "v2"));
  ASSERT_OK(Put(1, "mirko", "v3"));
  ASSERT_OK(Put(0, "foo", "v2"));
  ASSERT_OK(Put(2, "fodor", "v5"));
900
901
902
903
904
905
906
907
908
909
910

  for (int j = 0; j < 2; j++) {
    ReadOptions ro;
    std::vector<Iterator*> iterators;
    // Hold super version.
    if (j == 0) {
      ASSERT_OK(db_->NewIterators(ro, handles_, &iterators));
    }

    for (int i = 0; i < 3; ++i) {
      uint64_t max_total_in_memory_state =
911
          MaxTotalInMemoryState();
912
      Flush(i);
913
      AssertMaxTotalInMemoryState(max_total_in_memory_state);
914
915
916
917
918
919
920
    }
    ASSERT_OK(Put(1, "foofoo", "bar"));
    ASSERT_OK(Put(0, "foofoo", "bar"));

    for (auto* it : iterators) {
      delete it;
    }
921
  }
Igor Canadi's avatar
Igor Canadi committed
922
  Reopen();
923
924
925
926
927
928
929
930
931
932

  for (int iter = 0; iter <= 2; ++iter) {
    ASSERT_EQ("v2", Get(0, "foo"));
    ASSERT_EQ("v2", Get(0, "bar"));
    ASSERT_EQ("v3", Get(1, "mirko"));
    ASSERT_EQ("v5", Get(2, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(0, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
    ASSERT_EQ("NOT_FOUND", Get(2, "foo"));
    if (iter <= 1) {
Igor Canadi's avatar
Igor Canadi committed
933
      Reopen();
934
935
936
937
938
    }
  }
  Close();
}

939
// Makes sure that obsolete log files get deleted
Igor Sugak's avatar
Igor Sugak committed
940
TEST_F(ColumnFamilyTest, LogDeletionTest) {
Igor Canadi's avatar
Igor Canadi committed
941
  db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
942
  column_family_options_.arena_block_size = 4 * 1024;
943
  column_family_options_.write_buffer_size = 128000;  // 128KB
Igor Canadi's avatar
Igor Canadi committed
944
  Open();
945
946
947
948
  CreateColumnFamilies({"one", "two", "three", "four"});
  // Each bracket is one log file. if number is in (), it means
  // we don't need it anymore (it's been flushed)
  // []
949
  AssertCountLiveLogFiles(0);
950
  PutRandomData(0, 1, 128);
951
  // [0]
952
  PutRandomData(1, 1, 128);
953
  // [0, 1]
954
  PutRandomData(1, 1000, 128);
955
956
  WaitForFlush(1);
  // [0, (1)] [1]
957
  AssertCountLiveLogFiles(2);
958
  PutRandomData(0, 1, 128);
959
  // [0, (1)] [0, 1]
960
  AssertCountLiveLogFiles(2);
961
  PutRandomData(2, 1, 128);
962
  // [0, (1)] [0, 1, 2]
963
  PutRandomData(2, 1000, 128);
964
965
  WaitForFlush(2);
  // [0, (1)] [0, 1, (2)] [2]
966
  AssertCountLiveLogFiles(3);
967
  PutRandomData(2, 1000, 128);
968
969
  WaitForFlush(2);
  // [0, (1)] [0, 1, (2)] [(2)] [2]
970
  AssertCountLiveLogFiles(4);
971
  PutRandomData(3, 1, 128);
972
  // [0, (1)] [0, 1, (2)] [(2)] [2, 3]
973
  PutRandomData(1, 1, 128);
974
  // [0, (1)] [0, 1, (2)] [(2)] [1, 2, 3]
975
  AssertCountLiveLogFiles(4);
976
  PutRandomData(1, 1000, 128);
977
978
  WaitForFlush(1);
  // [0, (1)] [0, (1), (2)] [(2)] [(1), 2, 3] [1]
979
  AssertCountLiveLogFiles(5);
980
  PutRandomData(0, 1000, 128);
981
982
983
984
  WaitForFlush(0);
  // [(0), (1)] [(0), (1), (2)] [(2)] [(1), 2, 3] [1, (0)] [0]
  // delete obsolete logs -->
  // [(1), 2, 3] [1, (0)] [0]
985
  AssertCountLiveLogFiles(3);
986
  PutRandomData(0, 1000, 128);
987
988
  WaitForFlush(0);
  // [(1), 2, 3] [1, (0)], [(0)] [0]
989
  AssertCountLiveLogFiles(4);
990
  PutRandomData(1, 1000, 128);
991
992
  WaitForFlush(1);
  // [(1), 2, 3] [(1), (0)] [(0)] [0, (1)] [1]
993
  AssertCountLiveLogFiles(5);
994
  PutRandomData(2, 1000, 128);
995
996
  WaitForFlush(2);
  // [(1), (2), 3] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2]
997
  AssertCountLiveLogFiles(6);
998
  PutRandomData(3, 1000, 128);
999
1000
  WaitForFlush(3);
  // [(1), (2), (3)] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2, (3)] [3]
For faster browsing, not all history is shown. View entire blame