nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Add method to remove entry from language model dict content."

main
Keisuke Kuroyanagi 2014-08-18 03:58:25 +00:00 committed by Android (Google) Code Review
parent fd8f1cf1bd b4531d861e
commit 0db136e51f
5 changed files with 37 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
View File

@ -53,6 +53,16 @@ bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView pr
return mTrieMap.put(terminalId, probabilityEntry->encode(mHasHistoricalInfo), bitmapEntryIndex);
}
bool LanguageModelDictContent::removeNgramProbabilityEntry(const WordIdArrayView prevWordIds,
const int wordId) {
const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds);
if (bitmapEntryIndex == TrieMap::INVALID_INDEX) {
// Cannot find bitmap entry for the probability entry. The entry doesn't exist.
return false;
}
return mTrieMap.remove(wordId, bitmapEntryIndex);
}
bool LanguageModelDictContent::runGCInner(
const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const TrieMap::TrieMapRange trieMapRange,

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
View File

@ -61,12 +61,18 @@ class LanguageModelDictContent {
return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry);
}
bool removeProbabilityEntry(const int wordId) {
return removeNgramProbabilityEntry(WordIdArrayView(), wordId);
}
ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds,
const int wordId) const;
bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId,
const ProbabilityEntry *const probabilityEntry);
bool removeNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId);
private:
DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);

6
native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp
View File

@ -420,6 +420,10 @@ bool TrieMap::addNewEntryByExpandingTable(const uint32_t key, const uint64_t val
bool TrieMap::removeInner(const Entry &bitmapEntry) {
const int tableSize = popCount(bitmapEntry.getBitmap());
if (tableSize <= 0) {
// The table is empty. No need to remove any entries.
return true;
}
for (int i = 0; i < tableSize; ++i) {
const int entryIndex = bitmapEntry.getTableIndex() + i;
const Entry entry = readEntry(entryIndex);
@ -444,7 +448,7 @@ bool TrieMap::removeInner(const Entry &bitmapEntry) {
}
}
}
return freeTable(bitmapEntry.getTableIndex(), tableSize);
return true;
}
} // namespace latinime

14
native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp
View File

@ -35,6 +35,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbability) {
LanguageModelDictContent.getProbabilityEntry(wordId);
EXPECT_EQ(flag, entry.getFlags());
EXPECT_EQ(probability, entry.getProbability());
// Remove
EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));
EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
EXPECT_TRUE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
}
TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
@ -53,6 +60,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp());
EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel());
EXPECT_EQ(count, entry.getHistoricalInfo()->getCount());
// Remove
EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());
EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));
EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));
EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));
}
} // namespace

2
native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp
View File

@ -70,6 +70,8 @@ TEST(TrieMapTest, TestRemove) {
EXPECT_FALSE(result.mIsValid);
EXPECT_EQ(TrieMap::INVALID_INDEX, result.mNextLevelBitmapEntryIndex);
EXPECT_EQ(11ull, trieMap.getRoot(12).mValue);
EXPECT_TRUE(trieMap.putRoot(S_INT_MAX, 0xFFFFFFFFFull));
EXPECT_TRUE(trieMap.remove(S_INT_MAX, trieMap.getRootBitmapEntryIndex()));
}
TEST(TrieMapTest, TestSetAndGetLarge) {