Merge "Support removing n-gram entry in structure policy."
commit
3676c11472
|
@ -232,11 +232,9 @@ bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds
|
||||||
|
|
||||||
bool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWordIds,
|
bool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWordIds,
|
||||||
const int wordId) {
|
const int wordId) {
|
||||||
// TODO: Support n-gram.
|
|
||||||
LanguageModelDictContent *const languageModelDictContent =
|
LanguageModelDictContent *const languageModelDictContent =
|
||||||
mBuffers->getMutableLanguageModelDictContent();
|
mBuffers->getMutableLanguageModelDictContent();
|
||||||
return languageModelDictContent->removeNgramProbabilityEntry(prevWordIds.limit(1 /* maxSize */),
|
return languageModelDictContent->removeNgramProbabilityEntry(prevWordIds, wordId);
|
||||||
wordId);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Remove when we stop supporting v402 format.
|
// TODO: Remove when we stop supporting v402 format.
|
||||||
|
|
|
@ -375,15 +375,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
|
||||||
if (wordId == NOT_A_WORD_ID) {
|
if (wordId == NOT_A_WORD_ID) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
std::array<int, MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordsPtNodePos;
|
if (mNodeWriter.removeNgramEntry(prevWordIds, wordId)) {
|
||||||
for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
|
|
||||||
prevWordsPtNodePos[i] = mBuffers->getTerminalPositionLookupTable()
|
|
||||||
->getTerminalPtNodePosition(prevWordIds[i]);
|
|
||||||
}
|
|
||||||
const int wordPtNodePos = mBuffers->getTerminalPositionLookupTable()
|
|
||||||
->getTerminalPtNodePosition(wordId);
|
|
||||||
if (mUpdatingHelper.removeNgramEntry(WordIdArrayView::fromArray(prevWordsPtNodePos),
|
|
||||||
wordPtNodePos)) {
|
|
||||||
mBigramCount--;
|
mBigramCount--;
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in New Issue