Move methods of MultiBigramMap to cpp file.
Change-Id: Icf10795037a7e966ac843cd168fe45955b6aef56main
parent
11765ee804
commit
b685ffa13c
|
@ -30,4 +30,75 @@ const size_t MultiBigramMap::MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP = 25;
|
|||
// Most common previous word contexts currently have 100 bigrams
|
||||
const int MultiBigramMap::BigramMap::DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP = 100;
|
||||
|
||||
// Look up the bigram probability for the given word pair from the cached bigram maps.
|
||||
// Also caches the bigrams if there is space remaining and they have not been cached already.
|
||||
int MultiBigramMap::getBigramProbability(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||
const int wordPosition, const int nextWordPosition, const int unigramProbability) {
|
||||
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
|
||||
mBigramMaps.find(wordPosition);
|
||||
if (mapPosition != mBigramMaps.end()) {
|
||||
return mapPosition->second.getBigramProbability(structurePolicy, nextWordPosition,
|
||||
unigramProbability);
|
||||
}
|
||||
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
|
||||
addBigramsForWordPosition(structurePolicy, wordPosition);
|
||||
return mBigramMaps[wordPosition].getBigramProbability(structurePolicy,
|
||||
nextWordPosition, unigramProbability);
|
||||
}
|
||||
return readBigramProbabilityFromBinaryDictionary(structurePolicy, wordPosition,
|
||||
nextWordPosition, unigramProbability);
|
||||
}
|
||||
|
||||
void MultiBigramMap::BigramMap::init(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos) {
|
||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||
bigramsListPos);
|
||||
while (bigramsIt.hasNext()) {
|
||||
bigramsIt.next();
|
||||
if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) {
|
||||
continue;
|
||||
}
|
||||
mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability();
|
||||
mBloomFilter.setInFilter(bigramsIt.getBigramPos());
|
||||
}
|
||||
}
|
||||
|
||||
int MultiBigramMap::BigramMap::getBigramProbability(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||
const int nextWordPosition, const int unigramProbability) const {
|
||||
int bigramProbability = NOT_A_PROBABILITY;
|
||||
if (mBloomFilter.isInFilter(nextWordPosition)) {
|
||||
const hash_map_compat<int, int>::const_iterator bigramProbabilityIt =
|
||||
mBigramMap.find(nextWordPosition);
|
||||
if (bigramProbabilityIt != mBigramMap.end()) {
|
||||
bigramProbability = bigramProbabilityIt->second;
|
||||
}
|
||||
}
|
||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||
}
|
||||
|
||||
void MultiBigramMap::addBigramsForWordPosition(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position) {
|
||||
mBigramMaps[position].init(structurePolicy, position);
|
||||
}
|
||||
|
||||
int MultiBigramMap::readBigramProbabilityFromBinaryDictionary(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
|
||||
const int nextWordPosition, const int unigramProbability) {
|
||||
int bigramProbability = NOT_A_PROBABILITY;
|
||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||
bigramsListPos);
|
||||
while (bigramsIt.hasNext()) {
|
||||
bigramsIt.next();
|
||||
if (bigramsIt.getBigramPos() == nextWordPosition) {
|
||||
bigramProbability = bigramsIt.getProbability();
|
||||
break;
|
||||
}
|
||||
}
|
||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||
}
|
||||
|
||||
} // namespace latinime
|
||||
|
|
|
@ -38,21 +38,7 @@ class MultiBigramMap {
|
|||
// Look up the bigram probability for the given word pair from the cached bigram maps.
|
||||
// Also caches the bigrams if there is space remaining and they have not been cached already.
|
||||
int getBigramProbability(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||
const int wordPosition, const int nextWordPosition, const int unigramProbability) {
|
||||
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
|
||||
mBigramMaps.find(wordPosition);
|
||||
if (mapPosition != mBigramMaps.end()) {
|
||||
return mapPosition->second.getBigramProbability(structurePolicy, nextWordPosition,
|
||||
unigramProbability);
|
||||
}
|
||||
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
|
||||
addBigramsForWordPosition(structurePolicy, wordPosition);
|
||||
return mBigramMaps[wordPosition].getBigramProbability(structurePolicy,
|
||||
nextWordPosition, unigramProbability);
|
||||
}
|
||||
return readBigramProbabilityFromBinaryDictionary(structurePolicy, wordPosition,
|
||||
nextWordPosition, unigramProbability);
|
||||
}
|
||||
const int wordPosition, const int nextWordPosition, const int unigramProbability);
|
||||
|
||||
void clear() {
|
||||
mBigramMaps.clear();
|
||||
|
@ -67,33 +53,11 @@ class MultiBigramMap {
|
|||
~BigramMap() {}
|
||||
|
||||
void init(const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||
const int nodePos) {
|
||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||
bigramsListPos);
|
||||
while (bigramsIt.hasNext()) {
|
||||
bigramsIt.next();
|
||||
if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) {
|
||||
continue;
|
||||
}
|
||||
mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability();
|
||||
mBloomFilter.setInFilter(bigramsIt.getBigramPos());
|
||||
}
|
||||
}
|
||||
const int nodePos);
|
||||
|
||||
AK_FORCE_INLINE int getBigramProbability(
|
||||
int getBigramProbability(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy,
|
||||
const int nextWordPosition, const int unigramProbability) const {
|
||||
int bigramProbability = NOT_A_PROBABILITY;
|
||||
if (mBloomFilter.isInFilter(nextWordPosition)) {
|
||||
const hash_map_compat<int, int>::const_iterator bigramProbabilityIt =
|
||||
mBigramMap.find(nextWordPosition);
|
||||
if (bigramProbabilityIt != mBigramMap.end()) {
|
||||
bigramProbability = bigramProbabilityIt->second;
|
||||
}
|
||||
}
|
||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||
}
|
||||
const int nextWordPosition, const int unigramProbability) const;
|
||||
|
||||
private:
|
||||
// NOTE: The BigramMap class doesn't use DISALLOW_COPY_AND_ASSIGN() because its default
|
||||
|
@ -103,27 +67,12 @@ class MultiBigramMap {
|
|||
BloomFilter mBloomFilter;
|
||||
};
|
||||
|
||||
AK_FORCE_INLINE void addBigramsForWordPosition(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position) {
|
||||
mBigramMaps[position].init(structurePolicy, position);
|
||||
}
|
||||
void addBigramsForWordPosition(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int position);
|
||||
|
||||
AK_FORCE_INLINE int readBigramProbabilityFromBinaryDictionary(
|
||||
int readBigramProbabilityFromBinaryDictionary(
|
||||
const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
|
||||
const int nextWordPosition, const int unigramProbability) {
|
||||
int bigramProbability = NOT_A_PROBABILITY;
|
||||
const int bigramsListPos = structurePolicy->getBigramsPositionOfPtNode(nodePos);
|
||||
BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
|
||||
bigramsListPos);
|
||||
while (bigramsIt.hasNext()) {
|
||||
bigramsIt.next();
|
||||
if (bigramsIt.getBigramPos() == nextWordPosition) {
|
||||
bigramProbability = bigramsIt.getProbability();
|
||||
break;
|
||||
}
|
||||
}
|
||||
return structurePolicy->getProbability(unigramProbability, bigramProbability);
|
||||
}
|
||||
const int nextWordPosition, const int unigramProbability);
|
||||
|
||||
static const size_t MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP;
|
||||
hash_map_compat<int, BigramMap> mBigramMaps;
|
||||
|
|
Loading…
Reference in New Issue