提交 618bf638 编辑于 作者: mrambacher's avatar mrambacher 提交者: Facebook GitHub Bot
浏览文件

Add Functions to OptionTypeInfo (#6422)

Summary:
Added functions for parsing, serializing, and comparing elements to OptionTypeInfo.  These functions allow all of the special cases that could not be handled directly in the map of OptionTypeInfo to be moved into the map.  Using these functions, every type can be handled via the map rather than special cased.

By adding these functions, the code for handling options can become more standardized (fewer special cases) and (eventually) handled completely by common classes.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6422

Test Plan: pass make check

Reviewed By: siying

Differential Revision: D21269005

Pulled By: zhichao-cao

fbshipit-source-id: 9ba71c721a38ebf9ee88259d60bd81b3282b9077
上级 b810e62b
......@@ -518,6 +518,7 @@ find_package(Threads REQUIRED)
# Main library source code
set(SOURCES
cache/cache.cc
cache/clock_cache.cc
cache/lru_cache.cc
cache/sharded_cache.cc
......@@ -632,7 +633,6 @@ set(SOURCES
options/options.cc
options/options_helper.cc
options/options_parser.cc
options/options_sanity_check.cc
port/stack_trace.cc
table/adaptive/adaptive_table_factory.cc
table/block_based/binary_search_index_reader.cc
......
......@@ -112,6 +112,7 @@ ROCKSDB_OS_DEPS += ([(
cpp_library(
name = "rocksdb_lib",
srcs = [
"cache/cache.cc",
"cache/clock_cache.cc",
"cache/lru_cache.cc",
"cache/sharded_cache.cc",
......@@ -230,7 +231,6 @@ cpp_library(
"options/options.cc",
"options/options_helper.cc",
"options/options_parser.cc",
"options/options_sanity_check.cc",
"port/port_posix.cc",
"port/stack_trace.cc",
"table/adaptive/adaptive_table_factory.cc",
......
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/cache.h"
#include "cache/lru_cache.h"
#include "options/options_helper.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
Status Cache::CreateFromString(const ConfigOptions& /*opts*/,
const std::string& value,
std::shared_ptr<Cache>* result) {
Status status;
std::shared_ptr<Cache> cache;
if (value.find('=') == std::string::npos) {
cache = NewLRUCache(ParseSizeT(value));
} else {
#ifndef ROCKSDB_LITE
LRUCacheOptions cache_opts;
if (!ParseOptionHelper(reinterpret_cast<char*>(&cache_opts),
OptionType::kLRUCacheOptions, value)) {
status = Status::InvalidArgument("Invalid cache options");
}
cache = NewLRUCache(cache_opts);
#else
status = Status::NotSupported("Cannot load cache in LITE mode ", value);
#endif //! ROCKSDB_LITE
}
if (status.ok()) {
result->swap(cache);
}
return status;
}
} // namespace ROCKSDB_NAMESPACE
......@@ -40,7 +40,7 @@ class DBOptionsTest : public DBTestBase {
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map;
for (const auto opt : db_options_type_info) {
if (opt.second.IsMutable() && !opt.second.IsDeprecated()) {
if (opt.second.IsMutable() && opt.second.ShouldSerialize()) {
mutable_map[opt.first] = options_map[opt.first];
}
}
......@@ -58,7 +58,7 @@ class DBOptionsTest : public DBTestBase {
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map;
for (const auto opt : cf_options_type_info) {
if (opt.second.IsMutable() && !opt.second.IsDeprecated()) {
if (opt.second.IsMutable() && opt.second.ShouldSerialize()) {
mutable_map[opt.first] = options_map[opt.first];
}
}
......
......@@ -33,6 +33,7 @@
namespace ROCKSDB_NAMESPACE {
class Cache;
struct ConfigOptions;
extern const bool kDefaultToAdaptiveMutex;
......@@ -142,6 +143,21 @@ class Cache {
Cache(const Cache&) = delete;
Cache& operator=(const Cache&) = delete;
// Creates a new Cache based on the input value string and returns the result.
// Currently, this method can be used to create LRUCaches only
// @param config_options
// @param value The value might be:
// - an old-style cache ("1M") -- equivalent to NewLRUCache(1024*102(
// - Name-value option pairs -- "capacity=1M; num_shard_bits=4;
// For the LRUCache, the values are defined in LRUCacheOptions.
// @param result The new Cache object
// @return OK if the cache was sucessfully created
// @return NotFound if an invalid name was specified in the value
// @return InvalidArgument if either the options were not valid
static Status CreateFromString(const ConfigOptions& config_options,
const std::string& value,
std::shared_ptr<Cache>* result);
// Destroys all existing entries by calling the "deleter"
// function that was passed via the Insert() function.
//
......
......@@ -226,6 +226,12 @@ struct ConfigOptions {
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options".
// @return Status::NotFound means the one (or more) of the option name in
// the opts_map is not valid for this option
// @return Status::NotSupported means we do not know how to parse one of the
// value for this option
// @return Status::InvalidArgument means the one of the option values is not
// valid for this option.
Status GetColumnFamilyOptionsFromMap(
const ConfigOptions& config_options,
const ColumnFamilyOptions& base_options,
......@@ -268,6 +274,12 @@ Status GetColumnFamilyOptionsFromMap(
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options".
// @return Status::NotFound means the one (or more) of the option name in
// the opts_map is not valid for this option
// @return Status::NotSupported means we do not know how to parse one of the
// value for this option
// @return Status::InvalidArgument means the one of the option values is not
// valid for this option.
Status GetDBOptionsFromMap(
const ConfigOptions& cfg_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map,
......
......@@ -20,17 +20,20 @@
#pragma once
#include <stdlib.h>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>
#include "rocksdb/advanced_options.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
class Slice;
struct BlockBasedTableOptions;
struct ConfigOptions;
// A class that takes a bunch of keys, then generates filter
class FilterBitsBuilder {
......@@ -125,6 +128,18 @@ class FilterPolicy {
public:
virtual ~FilterPolicy();
// Creates a new FilterPolicy based on the input value string and returns the
// result The value might be an ID, and ID with properties, or an old-style
// policy string.
// The value describes the FilterPolicy being created.
// For BloomFilters, value may be a ":"-delimited value of the form:
// "bloomfilter:[bits_per_key]:[use_block_based_builder]",
// e.g. ""bloomfilter:4:true"
// The above string is equivalent to calling NewBloomFilterPolicy(4, true).
static Status CreateFromString(const ConfigOptions& config_options,
const std::string& value,
std::shared_ptr<const FilterPolicy>* result);
// Return the name of this policy. Note that if the filter encoding
// changes in an incompatible way, the name returned by this method
// must be changed. Otherwise, old incompatible filters may be
......
......@@ -47,8 +47,79 @@ int offset_of(T1 AdvancedColumnFamilyOptions::*member) {
size_t(&OptionsHelper::dummy_cf_options));
}
const std::string kNameComparator = "comparator";
const std::string kNameMergeOperator = "merge_operator";
static Status ParseCompressionOptions(const std::string& value,
const std::string& name,
CompressionOptions& compression_opts) {
size_t start = 0;
size_t end = value.find(':');
if (end == std::string::npos) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
compression_opts.window_bits = ParseInt(value.substr(start, end - start));
start = end + 1;
end = value.find(':', start);
if (end == std::string::npos) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
compression_opts.level = ParseInt(value.substr(start, end - start));
start = end + 1;
if (start >= value.size()) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
end = value.find(':', start);
compression_opts.strategy =
ParseInt(value.substr(start, value.size() - start));
// max_dict_bytes is optional for backwards compatibility
if (end != std::string::npos) {
start = end + 1;
if (start >= value.size()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.max_dict_bytes =
ParseInt(value.substr(start, value.size() - start));
end = value.find(':', start);
}
// zstd_max_train_bytes is optional for backwards compatibility
if (end != std::string::npos) {
start = end + 1;
if (start >= value.size()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.zstd_max_train_bytes =
ParseInt(value.substr(start, value.size() - start));
end = value.find(':', start);
}
// parallel_threads is optional for backwards compatibility
if (end != std::string::npos) {
start = end + 1;
if (start >= value.size()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.parallel_threads =
ParseInt(value.substr(start, value.size() - start));
end = value.find(':', start);
}
// enabled is optional for backwards compatibility
if (end != std::string::npos) {
start = end + 1;
if (start >= value.size()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.enabled =
ParseBoolean("", value.substr(start, value.size() - start));
}
return Status::OK();
}
const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
const std::string kOptNameCompOpts = "compression_opts";
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::cf_options_type_info = {
......@@ -280,9 +351,22 @@ std::unordered_map<std::string, OptionTypeInfo>
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, bottommost_compression)}},
{kNameComparator,
{"comparator",
{offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator,
OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}},
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose, 0,
// Parses the string and sets the corresponding comparator
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
auto old_comparator = reinterpret_cast<const Comparator**>(addr);
const Comparator* new_comparator = *old_comparator;
Status status = ObjectRegistry::NewInstance()->NewStaticObject(
value, &new_comparator);
if (status.ok()) {
*old_comparator = new_comparator;
return status;
}
return Status::OK();
}}},
{"prefix_extractor",
{offset_of(&ColumnFamilyOptions::prefix_extractor),
OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
......@@ -297,10 +381,74 @@ std::unordered_map<std::string, OptionTypeInfo>
{offset_of(&ColumnFamilyOptions::memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"memtable",
{offset_of(&ColumnFamilyOptions::memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kAlias,
OptionTypeFlags::kNone, 0,
// Parses the value string and updates the memtable_factory
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
std::unique_ptr<MemTableRepFactory> new_mem_factory;
Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory);
if (s.ok()) {
auto memtable_factory =
reinterpret_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
memtable_factory->reset(new_mem_factory.release());
}
return s;
}}},
{"table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kTableFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
OptionTypeFlags::kCompareLoose, 0}},
{"block_based_table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kTableFactory, OptionVerificationType::kAlias,
OptionTypeFlags::kCompareLoose, 0,
// Parses the input value and creates a BlockBasedTableFactory
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
// Nested options
auto old_table_factory =
reinterpret_cast<std::shared_ptr<TableFactory>*>(addr);
BlockBasedTableOptions table_opts, base_opts;
BlockBasedTableFactory* block_based_table_factory =
static_cast_with_check<BlockBasedTableFactory, TableFactory>(
old_table_factory->get());
if (block_based_table_factory != nullptr) {
base_opts = block_based_table_factory->table_options();
}
Status s = GetBlockBasedTableOptionsFromString(base_opts, value,
&table_opts);
if (s.ok()) {
old_table_factory->reset(NewBlockBasedTableFactory(table_opts));
}
return s;
}}},
{"plain_table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kTableFactory, OptionVerificationType::kAlias,
OptionTypeFlags::kCompareLoose, 0,
// Parses the input value and creates a PlainTableFactory
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
// Nested options
auto old_table_factory =
reinterpret_cast<std::shared_ptr<TableFactory>*>(addr);
PlainTableOptions table_opts, base_opts;
PlainTableFactory* plain_table_factory =
static_cast_with_check<PlainTableFactory, TableFactory>(
old_table_factory->get());
if (plain_table_factory != nullptr) {
base_opts = plain_table_factory->table_options();
}
Status s =
GetPlainTableOptionsFromString(base_opts, value, &table_opts);
if (s.ok()) {
old_table_factory->reset(NewPlainTableFactory(table_opts));
}
return s;
}}},
{"compaction_filter",
{offset_of(&ColumnFamilyOptions::compaction_filter),
OptionType::kCompactionFilter, OptionVerificationType::kByName,
......@@ -309,11 +457,19 @@ std::unordered_map<std::string, OptionTypeInfo>
{offset_of(&ColumnFamilyOptions::compaction_filter_factory),
OptionType::kCompactionFilterFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{kNameMergeOperator,
{"merge_operator",
{offset_of(&ColumnFamilyOptions::merge_operator),
OptionType::kMergeOperator,
OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kNone,
0}},
OptionVerificationType::kByNameAllowFromNull,
OptionTypeFlags::kCompareLoose, 0,
// Parses the input value as a MergeOperator, updating the value
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
auto mop = reinterpret_cast<std::shared_ptr<MergeOperator>*>(addr);
ObjectRegistry::NewInstance()->NewSharedObject<MergeOperator>(value,
mop);
return Status::OK();
}}},
{"compaction_style",
{offset_of(&ColumnFamilyOptions::compaction_style),
OptionType::kCompactionStyle, OptionVerificationType::kNormal,
......@@ -345,7 +501,36 @@ std::unordered_map<std::string, OptionTypeInfo>
{offset_of(&ColumnFamilyOptions::sample_for_compression),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, sample_for_compression)}}};
offsetof(struct MutableCFOptions, sample_for_compression)}},
// The following properties were handled as special cases in ParseOption
// This means that the properties could be read from the options file
// but never written to the file or compared to each other.
{kOptNameCompOpts,
{offset_of(&ColumnFamilyOptions::compression_opts),
OptionType::kUnknown, OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever |
OptionTypeFlags::kMutable),
offsetof(struct MutableCFOptions, compression_opts),
// Parses the value as a CompressionOptions
[](const ConfigOptions& /*opts*/, const std::string& name,
const std::string& value, char* addr) {
auto* compression = reinterpret_cast<CompressionOptions*>(addr);
return ParseCompressionOptions(value, name, *compression);
}}},
{kOptNameBMCompOpts,
{offset_of(&ColumnFamilyOptions::bottommost_compression_opts),
OptionType::kUnknown, OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever |
OptionTypeFlags::kMutable),
offsetof(struct MutableCFOptions, bottommost_compression_opts),
// Parses the value as a CompressionOptions
[](const ConfigOptions& /*opts*/, const std::string& name,
const std::string& value, char* addr) {
auto* compression = reinterpret_cast<CompressionOptions*>(addr);
return ParseCompressionOptions(value, name, *compression);
}}},
// End special case properties
};
Status ParseColumnFamilyOption(const ConfigOptions& config_options,
const std::string& name,
......@@ -355,104 +540,19 @@ Status ParseColumnFamilyOption(const ConfigOptions& config_options,
? UnescapeOptionString(org_value)
: org_value;
try {
if (name == "block_based_table_factory") {
// Nested options
BlockBasedTableOptions table_opt, base_table_options;
BlockBasedTableFactory* block_based_table_factory =
static_cast_with_check<BlockBasedTableFactory, TableFactory>(
new_options->table_factory.get());
if (block_based_table_factory != nullptr) {
base_table_options = block_based_table_factory->table_options();
}
Status table_opt_s = GetBlockBasedTableOptionsFromString(
base_table_options, value, &table_opt);
if (!table_opt_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
} else if (name == "plain_table_factory") {
// Nested options
PlainTableOptions table_opt, base_table_options;
PlainTableFactory* plain_table_factory =
static_cast_with_check<PlainTableFactory, TableFactory>(
new_options->table_factory.get());
if (plain_table_factory != nullptr) {
base_table_options = plain_table_factory->table_options();
}
Status table_opt_s =
GetPlainTableOptionsFromString(base_table_options, value, &table_opt);
if (!table_opt_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->table_factory.reset(NewPlainTableFactory(table_opt));
} else if (name == "memtable") {
std::unique_ptr<MemTableRepFactory> new_mem_factory;
Status mem_factory_s =
GetMemTableRepFactoryFromString(value, &new_mem_factory);
if (!mem_factory_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->memtable_factory.reset(new_mem_factory.release());
} else if (name == "bottommost_compression_opts") {
Status s = ParseCompressionOptions(
value, name, new_options->bottommost_compression_opts);
if (!s.ok()) {
return s;
}
} else if (name == "compression_opts") {
Status s =
ParseCompressionOptions(value, name, new_options->compression_opts);
if (!s.ok()) {
return s;
}
auto iter = cf_options_type_info.find(name);
if (iter == cf_options_type_info.end()) {
return Status::InvalidArgument(
"Unable to parse the specified CF option " + name);
} else {
if (name == kNameComparator) {
// Try to get comparator from object registry first.
// Only support static comparator for now.
Status status = ObjectRegistry::NewInstance()->NewStaticObject(
value, &new_options->comparator);
if (status.ok()) {
return status;
}
} else if (name == kNameMergeOperator) {
// Try to get merge operator from object registry first.
std::shared_ptr<MergeOperator> mo;
Status status =
ObjectRegistry::NewInstance()->NewSharedObject<MergeOperator>(
value, &new_options->merge_operator);
// Only support static comparator for now.
if (status.ok()) {
return status;
}
}
auto iter = cf_options_type_info.find(name);
if (iter == cf_options_type_info.end()) {
return Status::InvalidArgument(
"Unable to parse the specified CF option " + name);
}
const auto& opt_info = iter->second;
if (opt_info.IsDeprecated() ||
ParseOptionHelper(
reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return Status::OK();
} else if (opt_info.IsByName()) {
return Status::NotSupported("Deserializing the specified CF option " +
name + " is not supported");
} else {
return Status::InvalidArgument(
"Unable to parse the specified CF option " + name);
}
return iter->second.ParseOption(
config_options, name, value,
reinterpret_cast<char*>(new_options) + iter->second.offset);
}
} catch (const std::exception&) {
return Status::InvalidArgument("unable to parse the specified option " +
name);
}
return Status::OK();
}
#endif // ROCKSDB_LITE
......
......@@ -14,6 +14,7 @@
#include "rocksdb/cache.h"
#include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "rocksdb/rate_limiter.h"
#include "rocksdb/sst_file_manager.h"
#include "rocksdb/wal_filter.h"
......@@ -332,6 +333,37 @@ std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct DBOptions, best_efforts_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
// The following properties were handled as special cases in ParseOption
// This means that the properties could be read from the options file
// but never written to the file or compared to each other.
{"rate_limiter_bytes_per_sec",
{offsetof(struct DBOptions, rate_limiter), OptionType::kUnknown,
OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0,
// Parse the input value as a RateLimiter
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
auto limiter =
reinterpret_cast<std::shared_ptr<RateLimiter>*>(addr);
limiter->reset(NewGenericRateLimiter(
static_cast<int64_t>(ParseUint64(value))));
return Status::OK();
}}},
{"env",
{offsetof(struct DBOptions, env), OptionType::kUnknown,
OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0,
// Parse the input value as an Env
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) {
auto old_env = reinterpret_cast<Env**>(addr); // Get the old value
Env* new_env = *old_env; // Set new to old
Status s = Env::LoadEnv(value, &new_env); // Update new value
if (s.ok()) { // It worked
*old_env = new_env; // Update the old one
}
return s;
}}},
};
#endif // ROCKSDB_LITE
......
此差异已折叠。
......@@ -57,10 +57,6 @@ Status GetTableFactoryFromMap(
std::shared_ptr<TableFactory>* table_factory,
bool ignore_unknown_options = false);
Status ParseCompressionOptions(const std::string& value,
const std::string& name,
CompressionOptions& compression_opts);