db_filesnapshot.cc 7.47 KB
Newer Older
1
2
3
4
5
//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
6
7
// Copyright (c) 2012 Facebook.
// Use of this source code is governed by a BSD-style license that can be
Abhishek Kona's avatar
Abhishek Kona committed
8
// found in the LICENSE file.
9

Igor Canadi's avatar
Igor Canadi committed
10
11
#ifndef ROCKSDB_LITE

liuhuahang's avatar
liuhuahang committed
12
#ifndef __STDC_FORMAT_MACROS
Igor Canadi's avatar
Igor Canadi committed
13
#define __STDC_FORMAT_MACROS
liuhuahang's avatar
liuhuahang committed
14
15
#endif

Igor Canadi's avatar
Igor Canadi committed
16
#include <inttypes.h>
17
#include <algorithm>
18
19
#include <string>
#include <stdint.h>
20
21
#include "db/db_impl.h"
#include "db/filename.h"
Igor Canadi's avatar
Igor Canadi committed
22
#include "db/job_context.h"
23
#include "db/version_set.h"
24
25
#include "rocksdb/db.h"
#include "rocksdb/env.h"
26
27
#include "port/port.h"
#include "util/mutexlock.h"
28
#include "util/sync_point.h"
29
#include "util/file_util.h"
30

31
namespace rocksdb {
32
33
34

Status DBImpl::DisableFileDeletions() {
  MutexLock l(&mutex_);
35
36
  ++disable_delete_obsolete_files_;
  if (disable_delete_obsolete_files_ == 1) {
37
38
    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
        "File Deletions Disabled");
39
  } else {
40
    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
41
42
        "File Deletions Disabled, but already disabled. Counter: %d",
        disable_delete_obsolete_files_);
43
  }
44
45
46
  return Status::OK();
}

47
Status DBImpl::EnableFileDeletions(bool force) {
Igor Canadi's avatar
Igor Canadi committed
48
  JobContext job_context;
49
  bool should_purge_files = false;
Igor Canadi's avatar
Igor Canadi committed
50
51
  {
    MutexLock l(&mutex_);
52
53
54
55
56
57
58
    if (force) {
      // if force, we need to enable file deletions right away
      disable_delete_obsolete_files_ = 0;
    } else if (disable_delete_obsolete_files_ > 0) {
      --disable_delete_obsolete_files_;
    }
    if (disable_delete_obsolete_files_ == 0)  {
59
60
      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
          "File Deletions Enabled");
61
      should_purge_files = true;
Igor Canadi's avatar
Igor Canadi committed
62
      FindObsoleteFiles(&job_context, true);
63
    } else {
64
      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
65
66
          "File Deletions Enable, but not really enabled. Counter: %d",
          disable_delete_obsolete_files_);
67
68
69
    }
  }
  if (should_purge_files)  {
Igor Canadi's avatar
Igor Canadi committed
70
    PurgeObsoleteFiles(job_context);
Igor Canadi's avatar
Igor Canadi committed
71
  }
Igor Canadi's avatar
Igor Canadi committed
72
  job_context.Clean();
73
  LogFlush(db_options_.info_log);
74
75
76
  return Status::OK();
}

77
78
79
80
int DBImpl::IsFileDeletionsEnabled() const {
  return disable_delete_obsolete_files_;
}

Abhishek Kona's avatar
Abhishek Kona committed
81
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
82
83
                            uint64_t* manifest_file_size,
                            bool flush_memtable) {
84
85

  *manifest_file_size = 0;
86

87
88
  mutex_.Lock();

89
90
  if (flush_memtable) {
    // flush all dirty data to disk.
91
92
93
94
95
96
    Status status;
    for (auto cfd : *versions_->GetColumnFamilySet()) {
      cfd->Ref();
      mutex_.Unlock();
      status = FlushMemTable(cfd, FlushOptions());
      mutex_.Lock();
97
      cfd->Unref();
98
99
100
101
      if (!status.ok()) {
        break;
      }
    }
102
103
    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();

104
    if (!status.ok()) {
105
      mutex_.Unlock();
106
107
      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
          "Cannot Flush data %s\n", status.ToString().c_str());
108
109
      return status;
    }
110
111
112
  }

  // Make a set of all of the live *.sst files
113
  std::vector<FileDescriptor> live;
114
115
116
  for (auto cfd : *versions_->GetColumnFamilySet()) {
    cfd->current()->AddLiveFiles(&live);
  }
117

118
119
  ret.clear();
  ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
120
121
122

  // create names of the live files. The names are not absolute
  // paths, instead they are relative to dbname_;
123
  for (auto live_file : live) {
124
    ret.push_back(MakeTableFileName("", live_file.GetNumber()));
125
126
  }

127
  ret.push_back(CurrentFileName(""));
128
  ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
129

130
  // find length of manifest file while holding the mutex lock
131
  *manifest_file_size = versions_->manifest_file_size();
132

133
  mutex_.Unlock();
134
135
136
  return Status::OK();
}

137
Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
Igor Canadi's avatar
Igor Canadi committed
138
  return wal_manager_.GetSortedWalFiles(files);
139
}
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

// Builds an openable snapshot of RocksDB
Status DBImpl::CreateCheckpoint(const std::string& snapshot_dir) {
  Status s;
  std::vector<std::string> live_files;
  uint64_t manifest_file_size = 0;
  uint64_t sequence_number = GetLatestSequenceNumber();
  bool same_fs = true;

  if (env_->FileExists(snapshot_dir)) {
    return Status::InvalidArgument("Directory exists");
  }

  s = DisableFileDeletions();
  if (s.ok()) {
    // this will return live_files prefixed with "/"
    s = GetLiveFiles(live_files, &manifest_file_size, true);
  }
  if (!s.ok()) {
    EnableFileDeletions(false);
    return s;
  }

  Log(db_options_.info_log,
      "Started the snapshot process -- creating snapshot in directory %s",
      snapshot_dir.c_str());

  std::string full_private_path = snapshot_dir + ".tmp";

  // create snapshot directory
  s = env_->CreateDir(full_private_path);

  // copy/hard link live_files
  for (size_t i = 0; s.ok() && i < live_files.size(); ++i) {
    uint64_t number;
    FileType type;
    bool ok = ParseFileName(live_files[i], &number, &type);
    if (!ok) {
      s = Status::Corruption("Can't parse file name. This is very bad");
      break;
    }
    // we should only get sst, manifest and current files here
    assert(type == kTableFile || type == kDescriptorFile ||
           type == kCurrentFile);
    assert(live_files[i].size() > 0 && live_files[i][0] == '/');
    std::string src_fname = live_files[i];

    // rules:
    // * if it's kTableFile, then it's shared
    // * if it's kDescriptorFile, limit the size to manifest_file_size
    // * always copy if cross-device link
    if ((type == kTableFile) && same_fs) {
      Log(db_options_.info_log, "Hard Linking %s", src_fname.c_str());
      s = env_->LinkFile(GetName() + src_fname, full_private_path + src_fname);
      if (s.IsNotSupported()) {
        same_fs = false;
        s = Status::OK();
      }
    }
    if ((type != kTableFile) || (!same_fs)) {
      Log(db_options_.info_log, "Copying %s", src_fname.c_str());
      s = CopyFile(env_, GetName() + src_fname, full_private_path + src_fname,
                   (type == kDescriptorFile) ? manifest_file_size : 0);
    }
  }

  // we copied all the files, enable file deletions
  EnableFileDeletions(false);

  if (s.ok()) {
    // move tmp private backup to real snapshot directory
    s = env_->RenameFile(full_private_path, snapshot_dir);
  }
  if (s.ok()) {
    unique_ptr<Directory> snapshot_directory;
    env_->NewDirectory(snapshot_dir, &snapshot_directory);
    if (snapshot_directory != nullptr) {
      s = snapshot_directory->Fsync();
    }
  }

  if (!s.ok()) {
    // clean all the files we might have created
    Log(db_options_.info_log, "Snapshot failed -- %s", s.ToString().c_str());
    // we have to delete the dir and all its children
    std::vector<std::string> subchildren;
    env_->GetChildren(full_private_path, &subchildren);
    for (auto& subchild : subchildren) {
      Status s1 = env_->DeleteFile(full_private_path + subchild);
      if (s1.ok()) {
        Log(db_options_.info_log, "Deleted %s",
            (full_private_path + subchild).c_str());
      }
    }
    // finally delete the private dir
    Status s1 = env_->DeleteDir(full_private_path);
    Log(db_options_.info_log, "Deleted dir %s -- %s", full_private_path.c_str(),
        s1.ToString().c_str());
    return s;
  }

  // here we know that we succeeded and installed the new snapshot
  Log(db_options_.info_log, "Snapshot DONE. All is good");
  Log(db_options_.info_log, "Snapshot sequence number: %" PRIu64,
      sequence_number);

  return s;
}
248
}
Igor Canadi's avatar
Igor Canadi committed
249
250

#endif  // ROCKSDB_LITE