db_filesnapshot.cc 7.42 KB
Newer Older
1
2
3
4
5
//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
6
7
// Copyright (c) 2012 Facebook.
// Use of this source code is governed by a BSD-style license that can be
Abhishek Kona's avatar
Abhishek Kona committed
8
// found in the LICENSE file.
9

Igor Canadi's avatar
Igor Canadi committed
10
11
#ifndef ROCKSDB_LITE

liuhuahang's avatar
liuhuahang committed
12
#ifndef __STDC_FORMAT_MACROS
Igor Canadi's avatar
Igor Canadi committed
13
#define __STDC_FORMAT_MACROS
liuhuahang's avatar
liuhuahang committed
14
15
#endif

Igor Canadi's avatar
Igor Canadi committed
16
#include <inttypes.h>
17
#include <algorithm>
18
19
#include <string>
#include <stdint.h>
20
21
#include "db/db_impl.h"
#include "db/filename.h"
22
#include "db/version_set.h"
23
24
#include "rocksdb/db.h"
#include "rocksdb/env.h"
25
26
#include "port/port.h"
#include "util/mutexlock.h"
27
#include "util/sync_point.h"
28
#include "util/file_util.h"
29

30
namespace rocksdb {
31
32
33

Status DBImpl::DisableFileDeletions() {
  MutexLock l(&mutex_);
34
35
  ++disable_delete_obsolete_files_;
  if (disable_delete_obsolete_files_ == 1) {
36
37
    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
        "File Deletions Disabled");
38
  } else {
39
    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
40
41
        "File Deletions Disabled, but already disabled. Counter: %d",
        disable_delete_obsolete_files_);
42
  }
43
44
45
  return Status::OK();
}

46
Status DBImpl::EnableFileDeletions(bool force) {
Igor Canadi's avatar
Igor Canadi committed
47
  JobContext job_context;
48
  bool should_purge_files = false;
Igor Canadi's avatar
Igor Canadi committed
49
50
  {
    MutexLock l(&mutex_);
51
52
53
54
55
56
57
    if (force) {
      // if force, we need to enable file deletions right away
      disable_delete_obsolete_files_ = 0;
    } else if (disable_delete_obsolete_files_ > 0) {
      --disable_delete_obsolete_files_;
    }
    if (disable_delete_obsolete_files_ == 0)  {
58
59
      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
          "File Deletions Enabled");
60
      should_purge_files = true;
Igor Canadi's avatar
Igor Canadi committed
61
      FindObsoleteFiles(&job_context, true);
62
    } else {
63
      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
64
65
          "File Deletions Enable, but not really enabled. Counter: %d",
          disable_delete_obsolete_files_);
66
67
68
    }
  }
  if (should_purge_files)  {
Igor Canadi's avatar
Igor Canadi committed
69
    PurgeObsoleteFiles(job_context);
Igor Canadi's avatar
Igor Canadi committed
70
  }
71
  LogFlush(db_options_.info_log);
72
73
74
  return Status::OK();
}

75
76
77
78
int DBImpl::IsFileDeletionsEnabled() const {
  return disable_delete_obsolete_files_;
}

Abhishek Kona's avatar
Abhishek Kona committed
79
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
80
81
                            uint64_t* manifest_file_size,
                            bool flush_memtable) {
82
83

  *manifest_file_size = 0;
84

85
86
  mutex_.Lock();

87
88
  if (flush_memtable) {
    // flush all dirty data to disk.
89
90
91
92
93
94
    Status status;
    for (auto cfd : *versions_->GetColumnFamilySet()) {
      cfd->Ref();
      mutex_.Unlock();
      status = FlushMemTable(cfd, FlushOptions());
      mutex_.Lock();
95
      cfd->Unref();
96
97
98
99
      if (!status.ok()) {
        break;
      }
    }
100
101
    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();

102
    if (!status.ok()) {
103
      mutex_.Unlock();
104
105
      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
          "Cannot Flush data %s\n", status.ToString().c_str());
106
107
      return status;
    }
108
109
110
  }

  // Make a set of all of the live *.sst files
111
  std::vector<FileDescriptor> live;
112
113
114
  for (auto cfd : *versions_->GetColumnFamilySet()) {
    cfd->current()->AddLiveFiles(&live);
  }
115

116
117
  ret.clear();
  ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
118
119
120

  // create names of the live files. The names are not absolute
  // paths, instead they are relative to dbname_;
121
  for (auto live_file : live) {
122
    ret.push_back(MakeTableFileName("", live_file.GetNumber()));
123
124
  }

125
  ret.push_back(CurrentFileName(""));
126
  ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
127

128
  // find length of manifest file while holding the mutex lock
129
  *manifest_file_size = versions_->manifest_file_size();
130

131
  mutex_.Unlock();
132
133
134
  return Status::OK();
}

135
Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
Igor Canadi's avatar
Igor Canadi committed
136
  return wal_manager_.GetSortedWalFiles(files);
137
}
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

// Builds an openable snapshot of RocksDB
Status DBImpl::CreateCheckpoint(const std::string& snapshot_dir) {
  Status s;
  std::vector<std::string> live_files;
  uint64_t manifest_file_size = 0;
  uint64_t sequence_number = GetLatestSequenceNumber();
  bool same_fs = true;

  if (env_->FileExists(snapshot_dir)) {
    return Status::InvalidArgument("Directory exists");
  }

  s = DisableFileDeletions();
  if (s.ok()) {
    // this will return live_files prefixed with "/"
    s = GetLiveFiles(live_files, &manifest_file_size, true);
  }
  if (!s.ok()) {
    EnableFileDeletions(false);
    return s;
  }

  Log(db_options_.info_log,
      "Started the snapshot process -- creating snapshot in directory %s",
      snapshot_dir.c_str());

  std::string full_private_path = snapshot_dir + ".tmp";

  // create snapshot directory
  s = env_->CreateDir(full_private_path);

  // copy/hard link live_files
  for (size_t i = 0; s.ok() && i < live_files.size(); ++i) {
    uint64_t number;
    FileType type;
    bool ok = ParseFileName(live_files[i], &number, &type);
    if (!ok) {
      s = Status::Corruption("Can't parse file name. This is very bad");
      break;
    }
    // we should only get sst, manifest and current files here
    assert(type == kTableFile || type == kDescriptorFile ||
           type == kCurrentFile);
    assert(live_files[i].size() > 0 && live_files[i][0] == '/');
    std::string src_fname = live_files[i];

    // rules:
    // * if it's kTableFile, then it's shared
    // * if it's kDescriptorFile, limit the size to manifest_file_size
    // * always copy if cross-device link
    if ((type == kTableFile) && same_fs) {
      Log(db_options_.info_log, "Hard Linking %s", src_fname.c_str());
      s = env_->LinkFile(GetName() + src_fname, full_private_path + src_fname);
      if (s.IsNotSupported()) {
        same_fs = false;
        s = Status::OK();
      }
    }
    if ((type != kTableFile) || (!same_fs)) {
      Log(db_options_.info_log, "Copying %s", src_fname.c_str());
      s = CopyFile(env_, GetName() + src_fname, full_private_path + src_fname,
                   (type == kDescriptorFile) ? manifest_file_size : 0);
    }
  }

  // we copied all the files, enable file deletions
  EnableFileDeletions(false);

  if (s.ok()) {
    // move tmp private backup to real snapshot directory
    s = env_->RenameFile(full_private_path, snapshot_dir);
  }
  if (s.ok()) {
    unique_ptr<Directory> snapshot_directory;
    env_->NewDirectory(snapshot_dir, &snapshot_directory);
    if (snapshot_directory != nullptr) {
      s = snapshot_directory->Fsync();
    }
  }

  if (!s.ok()) {
    // clean all the files we might have created
    Log(db_options_.info_log, "Snapshot failed -- %s", s.ToString().c_str());
    // we have to delete the dir and all its children
    std::vector<std::string> subchildren;
    env_->GetChildren(full_private_path, &subchildren);
    for (auto& subchild : subchildren) {
      Status s1 = env_->DeleteFile(full_private_path + subchild);
      if (s1.ok()) {
        Log(db_options_.info_log, "Deleted %s",
            (full_private_path + subchild).c_str());
      }
    }
    // finally delete the private dir
    Status s1 = env_->DeleteDir(full_private_path);
    Log(db_options_.info_log, "Deleted dir %s -- %s", full_private_path.c_str(),
        s1.ToString().c_str());
    return s;
  }

  // here we know that we succeeded and installed the new snapshot
  Log(db_options_.info_log, "Snapshot DONE. All is good");
  Log(db_options_.info_log, "Snapshot sequence number: %" PRIu64,
      sequence_number);

  return s;
}
246
}
Igor Canadi's avatar
Igor Canadi committed
247
248

#endif  // ROCKSDB_LITE