Move methods of MultiBigramMap to cpp file.
Change-Id: Icf10795037a7e966ac843cd168fe45955b6aef56main
parent
11765ee804
commit
b685ffa13c
|
@ -30,4 +30,75 @@ const size_t MultiBigramMap::MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP = 25;
|
||||||
// Most common previous word contexts currently have 100 bigrams
|
// Most common previous word contexts currently have 100 bigrams
|
||||||
const int MultiBigramMap::BigramMap::DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP = 100;
|
const int MultiBigramMap::BigramMap::DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP = 100;
|
||||||
|
|
||||||
|
// Look up the bigram probability for the given word pair from the cached bigram maps.
|
||||||
|
// Also caches the bigrams if there is space remaining and they have not been cached already.
|
||||||
|
int MultiBigramMap::getBigramProbability(
|
||||||
|
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||||
|
const int wordPosition, const int nextWordPosition, const int unigramProbability) {
|
||||||
|
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
|
||||||
|
mBigramMaps.find(wordPosition);
|
||||||
|
if (mapPosition != mBigramMaps.end()) {
|
||||||
|
return mapPosition->second.getBigramProbability(structurePolicy, nextWordPosition,
|
||||||
|
unigramProbability);
|
||||||
|
}
|
||||||
|
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
|
||||||
|
addBigramsForWordPosition(structurePolicy, wordPosition);
|
||||||
|
return mBigramMaps[wordPosition].getBigramProbability(structurePolicy,
|
||||||
|
nextWordPosition, unigramProbability);
|
||||||
|
}
|
||||||
|
return readBigramProbabilityFromBinaryDictionary(structurePolicy, wordPosition,
|
||||||
|
nextWordPosition, unigramProbability);
|
||||||
|
}
|
||||||
|
|
||||||
|
void MultiBigramMap::BigramMap::init(
|
||||||
|
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos) {
|
||||||
|
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||||
|
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||||
|
bigramsListPos);
|
||||||
|
while (bigramsIt.hasNext()) {
|
||||||
|
bigramsIt.next();
|
||||||
|
if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability();
|
||||||
|
mBloomFilter.setInFilter(bigramsIt.getBigramPos());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int MultiBigramMap::BigramMap::getBigramProbability(
|
||||||
|
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||||
|
const int nextWordPosition, const int unigramProbability) const {
|
||||||
|
int bigramProbability = NOT_A_PROBABILITY;
|
||||||
|
if (mBloomFilter.isInFilter(nextWordPosition)) {
|
||||||
|
const hash_map_compat<int, int>::const_iterator bigramProbabilityIt =
|
||||||
|
mBigramMap.find(nextWordPosition);
|
||||||
|
if (bigramProbabilityIt != mBigramMap.end()) {
|
||||||
|
bigramProbability = bigramProbabilityIt->second;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||||
|
}
|
||||||
|
|
||||||
|
void MultiBigramMap::addBigramsForWordPosition(
|
||||||
|
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position) {
|
||||||
|
mBigramMaps[position].init(structurePolicy, position);
|
||||||
|
}
|
||||||
|
|
||||||
|
int MultiBigramMap::readBigramProbabilityFromBinaryDictionary(
|
||||||
|
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
|
||||||
|
const int nextWordPosition, const int unigramProbability) {
|
||||||
|
int bigramProbability = NOT_A_PROBABILITY;
|
||||||
|
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||||
|
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||||
|
bigramsListPos);
|
||||||
|
while (bigramsIt.hasNext()) {
|
||||||
|
bigramsIt.next();
|
||||||
|
if (bigramsIt.getBigramPos() == nextWordPosition) {
|
||||||
|
bigramProbability = bigramsIt.getProbability();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
|
|
|
@ -38,21 +38,7 @@ class MultiBigramMap {
|
||||||
// Look up the bigram probability for the given word pair from the cached bigram maps.
|
// Look up the bigram probability for the given word pair from the cached bigram maps.
|
||||||
// Also caches the bigrams if there is space remaining and they have not been cached already.
|
// Also caches the bigrams if there is space remaining and they have not been cached already.
|
||||||
int getBigramProbability(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
int getBigramProbability(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||||
const int wordPosition, const int nextWordPosition, const int unigramProbability) {
|
const int wordPosition, const int nextWordPosition, const int unigramProbability);
|
||||||
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
|
|
||||||
mBigramMaps.find(wordPosition);
|
|
||||||
if (mapPosition != mBigramMaps.end()) {
|
|
||||||
return mapPosition->second.getBigramProbability(structurePolicy, nextWordPosition,
|
|
||||||
unigramProbability);
|
|
||||||
}
|
|
||||||
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
|
|
||||||
addBigramsForWordPosition(structurePolicy, wordPosition);
|
|
||||||
return mBigramMaps[wordPosition].getBigramProbability(structurePolicy,
|
|
||||||
nextWordPosition, unigramProbability);
|
|
||||||
}
|
|
||||||
return readBigramProbabilityFromBinaryDictionary(structurePolicy, wordPosition,
|
|
||||||
nextWordPosition, unigramProbability);
|
|
||||||
}
|
|
||||||
|
|
||||||
void clear() {
|
void clear() {
|
||||||
mBigramMaps.clear();
|
mBigramMaps.clear();
|
||||||
|
@ -67,33 +53,11 @@ class MultiBigramMap {
|
||||||
~BigramMap() {}
|
~BigramMap() {}
|
||||||
|
|
||||||
void init(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
void init(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||||
const int nodePos) {
|
const int nodePos);
|
||||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
|
||||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
|
||||||
bigramsListPos);
|
|
||||||
while (bigramsIt.hasNext()) {
|
|
||||||
bigramsIt.next();
|
|
||||||
if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability();
|
|
||||||
mBloomFilter.setInFilter(bigramsIt.getBigramPos());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE int getBigramProbability(
|
int getBigramProbability(
|
||||||
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||||
const int nextWordPosition, const int unigramProbability) const {
|
const int nextWordPosition, const int unigramProbability) const;
|
||||||
int bigramProbability = NOT_A_PROBABILITY;
|
|
||||||
if (mBloomFilter.isInFilter(nextWordPosition)) {
|
|
||||||
const hash_map_compat<int, int>::const_iterator bigramProbabilityIt =
|
|
||||||
mBigramMap.find(nextWordPosition);
|
|
||||||
if (bigramProbabilityIt != mBigramMap.end()) {
|
|
||||||
bigramProbability = bigramProbabilityIt->second;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// NOTE: The BigramMap class doesn't use DISALLOW_COPY_AND_ASSIGN() because its default
|
// NOTE: The BigramMap class doesn't use DISALLOW_COPY_AND_ASSIGN() because its default
|
||||||
|
@ -103,27 +67,12 @@ class MultiBigramMap {
|
||||||
BloomFilter mBloomFilter;
|
BloomFilter mBloomFilter;
|
||||||
};
|
};
|
||||||
|
|
||||||
AK_FORCE_INLINE void addBigramsForWordPosition(
|
void addBigramsForWordPosition(
|
||||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position) {
|
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position);
|
||||||
mBigramMaps[position].init(structurePolicy, position);
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE int readBigramProbabilityFromBinaryDictionary(
|
int readBigramProbabilityFromBinaryDictionary(
|
||||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
|
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
|
||||||
const int nextWordPosition, const int unigramProbability) {
|
const int nextWordPosition, const int unigramProbability);
|
||||||
int bigramProbability = NOT_A_PROBABILITY;
|
|
||||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
|
||||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
|
||||||
bigramsListPos);
|
|
||||||
while (bigramsIt.hasNext()) {
|
|
||||||
bigramsIt.next();
|
|
||||||
if (bigramsIt.getBigramPos() == nextWordPosition) {
|
|
||||||
bigramProbability = bigramsIt.getProbability();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
|
||||||
}
|
|
||||||
|
|
||||||
static const size_t MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP;
|
static const size_t MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP;
|
||||||
hash_map_compat<int, BigramMap> mBigramMaps;
|
hash_map_compat<int, BigramMap> mBigramMaps;
|
||||||
|
|
Loading…
Reference in New Issue