Use bitset for BloomFilter.

Before:
(0)  660.00 (1.43%)
(1)  45320.00 (98.18%)
(2)  80.00 (0.17%)
Total 46160.00 (sum of others 46060.00)

After:
(0)  620.00 (1.34%)
(1)  45310.00 (98.05%)
(2)  130.00 (0.28%)
Total 46210.00 (sum of others 46060.00)

Change-Id: I936b639c50e15208aee999a929b33983c6caa59d
main
Keisuke Kuroyanagi 2014-04-09 16:18:23 +09:00
parent 113523d22d
commit 2fd0bf9a37
3 changed files with 18 additions and 49 deletions

View File

@ -27,7 +27,6 @@ LATIN_IME_CORE_SRC_FILES := \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
bigram_dictionary.cpp \
bloom_filter.cpp \
dictionary.cpp \
digraph_utils.cpp \
error_type_utils.cpp \

View File

@ -1,25 +0,0 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/core/dictionary/bloom_filter.h"
namespace latinime {
// Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest
// prime under 128 * 8.
const int BloomFilter::BIGRAM_FILTER_MODULO = 1021;
} // namespace latinime

View File

@ -17,8 +17,7 @@
#ifndef LATINIME_BLOOM_FILTER_H
#define LATINIME_BLOOM_FILTER_H
#include <cstdint>
#include <cstring>
#include <bitset>
#include "defines.h"
@ -34,41 +33,37 @@ namespace latinime {
// Total 148603.14 (sum of others 148579.90)
class BloomFilter {
public:
BloomFilter() {
ASSERT(BIGRAM_FILTER_BYTE_SIZE * 8 >= BIGRAM_FILTER_MODULO);
memset(mFilter, 0, sizeof(mFilter));
BloomFilter() : mFilter() {}
AK_FORCE_INLINE void setInFilter(const int position) {
mFilter.set(getIndex(position));
}
// TODO: uint32_t position
AK_FORCE_INLINE void setInFilter(const int32_t position) {
const uint32_t bucket = static_cast<uint32_t>(position % BIGRAM_FILTER_MODULO);
mFilter[bucket >> 3] |= static_cast<uint8_t>(1 << (bucket & 0x7));
}
// TODO: uint32_t position
AK_FORCE_INLINE bool isInFilter(const int32_t position) const {
const uint32_t bucket = static_cast<uint32_t>(position % BIGRAM_FILTER_MODULO);
return (mFilter[bucket >> 3] & static_cast<uint8_t>(1 << (bucket & 0x7))) != 0;
AK_FORCE_INLINE bool isInFilter(const int position) const {
return mFilter.test(getIndex(position));
}
private:
DISALLOW_ASSIGNMENT_OPERATOR(BloomFilter);
// Size, in bytes, of the bloom filter index for bigrams
// 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k,
AK_FORCE_INLINE size_t getIndex(const int position) const {
return static_cast<size_t>(position) % BIGRAM_FILTER_MODULO;
}
// Size, in bits, of the bloom filter index for bigrams
// The probability of false positive is (1 - e ** (-kn/m))**k,
// where k is the number of hash functions, n the number of bigrams, and m the number of
// bits we can test.
// At the moment 100 is the maximum number of bigrams for a word with the current
// At the moment 100 is the maximum number of bigrams for a word with the current main
// dictionaries, so n = 100. 1024 buckets give us m = 1024.
// With 1 hash function, our false positive rate is about 9.3%, which should be enough for
// our uses since we are only using this to increase average performance. For the record,
// k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%,
// and m = 4096 gives 2.4%.
// This is assigned here because it is used for array size.
static const int BIGRAM_FILTER_BYTE_SIZE = 128;
static const int BIGRAM_FILTER_MODULO;
uint8_t mFilter[BIGRAM_FILTER_BYTE_SIZE];
// This is assigned here because it is used for bitset size.
// 1021 is the largest prime under 1024.
static const size_t BIGRAM_FILTER_MODULO = 1021;
std::bitset<BIGRAM_FILTER_MODULO> mFilter;
};
} // namespace latinime
#endif // LATINIME_BLOOM_FILTER_H