Bloom Filter
bloom filter是leveldb中用来在一个block中检测key是否存在的工具，主要在BloomFilterPolicy类中实现：
class BloomFilterPolicy : public FilterPolicy
{
private:
size_t bits_per_key_;
size_t k_;
public:
explicit BloomFilterPolicy(int bits_per_key)
: bits_per_key_(bits_per_key)
{
// ...
}
// ...
virtual void CreateFilter(const Slice *keys, int n, std::string *dst) const
{
// ...
}
virtual bool KeyMayMatch(const Slice &key, const Slice &bloom_filter) const
{
// ...
}
};其中，bits_per_key_表示m/n，即bloom filter中包含的位数组长度与sstable中key的数量的比值。k_为hash函数的个数。
接下来看BloomFilterPolicy的构造函数，因为bloom filter的FPR（false positive rate）为(1-e^(-kn/m))^k，当k为log(2)(m/n)时，FPR最小，因此这里用bits_per_key_乘以0.69来求得k_的值，当然最后还是要将k限定在1-30的范围内：
// We intentionally round down to reduce probing cost a little bit
k_ = static_cast<size_t>(bits_per_key * 0.69); // 0.69 =~ ln(2)
if (k_ < 1)
k_ = 1;
if (k_ > 30)
k_ = 30;通过调用CreateFilter函数创建一个新的bloom filter。这个函数首先计算出需要多少字节来存储这个新的bloom filter：
// Compute bloom filter size (in both bits and bytes)
size_t bits = n * bits_per_key_;
// For small n, we can see a very high false positive rate. Fix it
// by enforcing a minimum bloom filter length.
if (bits < 64)
bits = 64;
size_t bytes = (bits + 7) / 8;
bits = bytes * 8;接下来函数将存储bloom filter的字符串dst进行扩展，使其能够存储刚刚计算出的新增的bytes个字节，dst中原来存储的是之前得到的bloom filter，同时将当前正在构建的这个bloom filter使用的hash函数的个数存入dst的末尾：
const size_t init_size = dst->size();
dst->resize(init_size + bytes, 0);
dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter最后依次取出key，通过double hash来计算出key的k_个hash后的结果，并将值存入dst指向的字符串中：
char *array = &(*dst)[init_size];
for (int i = 0; i < n; i++)
{
// Use double-hashing to generate a sequence of hash values.
// See analysis in [Kirsch,Mitzenmacher 2006].
uint32_t h = BloomHash(keys[i]);
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
for (size_t j = 0; j < k_; j++)
{
const uint32_t bitpos = h % bits;
array[bitpos / 8] |= (1 << (bitpos % 8));
h += delta;
}
}这里double hash的算法为：
Hi = h1(x) + ih2(x)
h2(x) = (h1(x) >> 17) | (h1(x) << 15)通过调用KeyMayMatch函数可以判断一个key是不是在bloom filter表示的key集合中。这个函数首先得到给定bloom filter的一些相关信息，比如bit数、hash函数的个数（生成一个bloom filter时，hash函数的个数被编码在bloom filter的最后一个字节）等，如果hash函数的个数大于30，则直接返回true：
const size_t len = bloom_filter.size();
if (len < 2)
return false;
const char *array = bloom_filter.data();
const size_t bits = (len - 1) * 8;
// Use the encoded k so that we can read filters generated by
// bloom filters created using different parameters.
const size_t k = array[len - 1];
if (k > 30)
{
// Reserved for potentially new encodings for short bloom filters.
// Consider it a match.
return true;
}然后通过依次计算key的k个hash函数的值，判断这个key是否在bloom filter表示的key集合中：
uint32_t h = BloomHash(key);
const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
for (size_t j = 0; j < k; j++)
{
const uint32_t bitpos = h % bits;
if ((array[bitpos / 8] & (1 << (bitpos % 8))) == 0)
return false;
h += delta;
}
return true;LevelDB还封装了FilterBlockBuilder类来构建在sstable中存储bloom filter的block：
// A FilterBlockBuilder is used to construct all of the filters for a
// particular Table. It generates a single string which is stored as
// a special block in the Table.
//
// The sequence of calls to FilterBlockBuilder must match the regexp:
// (StartBlock AddKey*)* Finish
class FilterBlockBuilder
{
public:
explicit FilterBlockBuilder(const FilterPolicy *);
void StartBlock(uint64_t block_offset);
void AddKey(const Slice &key);
Slice Finish();
private:
void GenerateFilter();
const FilterPolicy *policy_;
std::string keys_; // Flattened key contents
std::vector<size_t> start_; // Starting index in keys_ of each key
std::string result_; // Filter data computed so far
std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
std::vector<uint32_t> filter_offsets_;
// ...
};keys_用来存储需要添加进bloom filter的key，每个key都简单的连接在keys_的后面，start_存储每个key在keys_中的偏移量，result_为当前已经计算得到的bloom filter的值，tmp_keys_用于在调用CreateFilter函数时传递参数，filter_offsets_用于存储构建的每一个bloom filter在当前filter block中的偏移量。
StartBlock函数在开始构建bloom filter前调用。因为bloom filter的构建策略是每2k个字节就构建一个bloom filter，所以StartBlock函数会首先判断当前sstable中尚未构建bloom filter的数据是否已经达到了2k，如果达到，就调用GenerateFilter()函数为sstable中尚未构建bloom filter的数据构建新的bloom filter。
void FilterBlockBuilder::StartBlock(uint64_t block_offset)
{
uint64_t filter_index = (block_offset / kFilterBase);
assert(filter_index >= filter_offsets_.size());
while (filter_index > filter_offsets_.size())
{
GenerateFilter();
}
}其中GenerateFilter函数为：
void FilterBlockBuilder::GenerateFilter()函数首先计算出需要添加进bloom filter中的key的数量，并将key存入tmp_keys_以便传参：
const size_t num_keys = start_.size();
if (num_keys == 0)
{
// Fast path if there are no keys for this filter
filter_offsets_.push_back(result_.size());
return;
}
// Make list of keys from flattened key structure
start_.push_back(keys_.size()); // Simplify length computation
tmp_keys_.resize(num_keys);
for (size_t i = 0; i < num_keys; i++)
{
const char *base = keys_.data() + start_[i];
size_t length = start_[i + 1] - start_[i];
tmp_keys_[i] = Slice(base, length);
}接下来将新的bloom filter在filter block中的偏移量存入filter_offsets_，然后调用BloomFilterPolicy类的CreateFilter函数构建新的bloom filter：
// Generate filter for current set of keys and append to result_.
filter_offsets_.push_back(result_.size());
policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_);AddKey函数比较简单，只是将key加在keys_的后面，然后在start_中添加这个key的偏移量：
void FilterBlockBuilder::AddKey(const Slice &key)
{
Slice k = key;
start_.push_back(keys_.size());
keys_.append(k.data(), k.size());
}Finish函数完成一个filter block的构建
Slice FilterBlockBuilder::Finish()首先，当FilterBlockBuilder中还有key尚未构建bloom filter时，调用GenerateFilter函数将其构建为一个bloom filter：
if (!start_.empty())
{
GenerateFilter();
}然后将每个bloom filter在result_中的偏移量编码后加入result_：
// Append array of per-filter offsets
const uint32_t array_offset = result_.size();
for (size_t i = 0; i < filter_offsets_.size(); i++)
{
PutFixed32(&result_, filter_offsets_[i]);
}最后将描述bloom filter在result_中的偏移量的数组在result_中的偏移量编码后存入result_，这里还要存储kFilterBaseLg（sstable中每1 << kFilterBaseLg个字节构建一个bloom filter）：
PutFixed32(&result_, array_offset);
result_.push_back(kFilterBaseLg); // Save encoding parameter in result
return Slice(result_);LevelDB同时还封装了一个FilterBlockReader类用于filter block的读取：
class FilterBlockReader
{
public:
// REQUIRES: "contents" and *policy must stay live while *this is live.
FilterBlockReader(const FilterPolicy *policy, const Slice &contents);
bool KeyMayMatch(uint64_t block_offset, const Slice &key);
private:
const FilterPolicy *policy_;
const char *data_; // Pointer to filter data (at block-start)
const char *offset_; // Pointer to beginning of offset array (at block-end)
size_t num_; // Number of entries in offset array
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
};其中data_指向bloom filter的起始位置，offset_指向描述bloom filter在filter block中的偏移量的数组的起始位置，num_表示offset_数组的元素个数，base_lg_即为之前提到的kFilterBaseLg。
这里调用KeyMayMatch函数来实现对给定key是否在给定block中的检查：
bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice &key)
{
}函数首先根据给定block的block_offset找到该block对应的是filter block中的第几个bloom filter：
uint64_t index = block_offset >> base_lg_;然后得到对应的bloom filter的起始位置和结束位置，并调用BloomFilterPolicy类的KeyMayMatch函数进行检查：
if (index < num_)
{
uint32_t start = DecodeFixed32(offset_ + index * 4);
uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4);
if (start <= limit && limit <= static_cast<size_t>(offset_ - data_))
{
Slice filter = Slice(data_ + start, limit - start);
return policy_->KeyMayMatch(key, filter);
}
else if (start == limit)
{
// Empty filters do not match any keys
return false;
}
}
return true; // Errors are treated as potential matches239 Love u