am 7e9869a2: Merge "Support adding n-gram entry in structure policy."
* commit '7e9869a2aef3987a22426f51198eb04ff08a32c2': Support adding n-gram entry in structure policy.main
commit
54b450bb7b
|
@ -43,10 +43,12 @@ class DynamicPtUpdatingHelper {
|
|||
const int *const wordCodePoints, const int codePointCount,
|
||||
const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram);
|
||||
|
||||
// TODO: Remove after stopping supporting v402.
|
||||
// Add an n-gram entry.
|
||||
bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos,
|
||||
const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
|
||||
|
||||
// TODO: Remove after stopping supporting v402.
|
||||
// Remove an n-gram entry.
|
||||
bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos);
|
||||
|
||||
|
|
|
@ -211,19 +211,17 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
|
|||
|
||||
bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
|
||||
const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
|
||||
// TODO: Support n-gram.
|
||||
LanguageModelDictContent *const languageModelDictContent =
|
||||
mBuffers->getMutableLanguageModelDictContent();
|
||||
const ProbabilityEntry probabilityEntry =
|
||||
languageModelDictContent->getNgramProbabilityEntry(
|
||||
prevWordIds.limit(1 /* maxSize */), wordId);
|
||||
languageModelDictContent->getNgramProbabilityEntry(prevWordIds, wordId);
|
||||
const ProbabilityEntry probabilityEntryOfBigramProperty(bigramProperty);
|
||||
const ProbabilityEntry updatedProbabilityEntry = createUpdatedEntryFrom(
|
||||
&probabilityEntry, &probabilityEntryOfBigramProperty);
|
||||
if (!languageModelDictContent->setNgramProbabilityEntry(
|
||||
prevWordIds.limit(1 /* maxSize */), wordId, &updatedProbabilityEntry)) {
|
||||
AKLOGE("Cannot add new ngram entry. prevWordId: %d, wordId: %d",
|
||||
prevWordIds[0], wordId);
|
||||
prevWordIds, wordId, &updatedProbabilityEntry)) {
|
||||
AKLOGE("Cannot add new ngram entry. prevWordId[0]: %d, prevWordId.size(): %zd, wordId: %d",
|
||||
prevWordIds[0], prevWordIds.size(), wordId);
|
||||
return false;
|
||||
}
|
||||
if (!probabilityEntry.isValid() && outAddedNewBigram) {
|
||||
|
|
|
@ -335,17 +335,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
|
|||
if (wordId == NOT_A_WORD_ID) {
|
||||
return false;
|
||||
}
|
||||
// TODO: Support N-gram.
|
||||
bool addedNewEntry = false;
|
||||
WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordsPtNodePos;
|
||||
for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
|
||||
prevWordsPtNodePos[i] = mBuffers->getTerminalPositionLookupTable()
|
||||
->getTerminalPtNodePosition(prevWordIds[i]);
|
||||
}
|
||||
const int wordPtNodePos = mBuffers->getTerminalPositionLookupTable()
|
||||
->getTerminalPtNodePosition(wordId);
|
||||
if (mUpdatingHelper.addNgramEntry(WordIdArrayView::fromArray(prevWordsPtNodePos),
|
||||
wordPtNodePos, bigramProperty, &addedNewEntry)) {
|
||||
if (mNodeWriter.addNgramEntry(prevWordIds, wordId, bigramProperty, &addedNewEntry)) {
|
||||
if (addedNewEntry) {
|
||||
mBigramCount++;
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue