am 7e9869a2: Merge "Support adding n-gram entry in structure policy."
* commit '7e9869a2aef3987a22426f51198eb04ff08a32c2': Support adding n-gram entry in structure policy.main
commit
54b450bb7b
|
@ -43,10 +43,12 @@ class DynamicPtUpdatingHelper {
|
||||||
const int *const wordCodePoints, const int codePointCount,
|
const int *const wordCodePoints, const int codePointCount,
|
||||||
const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram);
|
const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram);
|
||||||
|
|
||||||
|
// TODO: Remove after stopping supporting v402.
|
||||||
// Add an n-gram entry.
|
// Add an n-gram entry.
|
||||||
bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos,
|
bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos,
|
||||||
const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
|
const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
|
||||||
|
|
||||||
|
// TODO: Remove after stopping supporting v402.
|
||||||
// Remove an n-gram entry.
|
// Remove an n-gram entry.
|
||||||
bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos);
|
bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos);
|
||||||
|
|
||||||
|
|
|
@ -211,19 +211,17 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
|
||||||
|
|
||||||
bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
|
bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
|
||||||
const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
|
const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
|
||||||
// TODO: Support n-gram.
|
|
||||||
LanguageModelDictContent *const languageModelDictContent =
|
LanguageModelDictContent *const languageModelDictContent =
|
||||||
mBuffers->getMutableLanguageModelDictContent();
|
mBuffers->getMutableLanguageModelDictContent();
|
||||||
const ProbabilityEntry probabilityEntry =
|
const ProbabilityEntry probabilityEntry =
|
||||||
languageModelDictContent->getNgramProbabilityEntry(
|
languageModelDictContent->getNgramProbabilityEntry(prevWordIds, wordId);
|
||||||
prevWordIds.limit(1 /* maxSize */), wordId);
|
|
||||||
const ProbabilityEntry probabilityEntryOfBigramProperty(bigramProperty);
|
const ProbabilityEntry probabilityEntryOfBigramProperty(bigramProperty);
|
||||||
const ProbabilityEntry updatedProbabilityEntry = createUpdatedEntryFrom(
|
const ProbabilityEntry updatedProbabilityEntry = createUpdatedEntryFrom(
|
||||||
&probabilityEntry, &probabilityEntryOfBigramProperty);
|
&probabilityEntry, &probabilityEntryOfBigramProperty);
|
||||||
if (!languageModelDictContent->setNgramProbabilityEntry(
|
if (!languageModelDictContent->setNgramProbabilityEntry(
|
||||||
prevWordIds.limit(1 /* maxSize */), wordId, &updatedProbabilityEntry)) {
|
prevWordIds, wordId, &updatedProbabilityEntry)) {
|
||||||
AKLOGE("Cannot add new ngram entry. prevWordId: %d, wordId: %d",
|
AKLOGE("Cannot add new ngram entry. prevWordId[0]: %d, prevWordId.size(): %zd, wordId: %d",
|
||||||
prevWordIds[0], wordId);
|
prevWordIds[0], prevWordIds.size(), wordId);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!probabilityEntry.isValid() && outAddedNewBigram) {
|
if (!probabilityEntry.isValid() && outAddedNewBigram) {
|
||||||
|
|
|
@ -335,17 +335,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
|
||||||
if (wordId == NOT_A_WORD_ID) {
|
if (wordId == NOT_A_WORD_ID) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// TODO: Support N-gram.
|
|
||||||
bool addedNewEntry = false;
|
bool addedNewEntry = false;
|
||||||
WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordsPtNodePos;
|
if (mNodeWriter.addNgramEntry(prevWordIds, wordId, bigramProperty, &addedNewEntry)) {
|
||||||
for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
|
|
||||||
prevWordsPtNodePos[i] = mBuffers->getTerminalPositionLookupTable()
|
|
||||||
->getTerminalPtNodePosition(prevWordIds[i]);
|
|
||||||
}
|
|
||||||
const int wordPtNodePos = mBuffers->getTerminalPositionLookupTable()
|
|
||||||
->getTerminalPtNodePosition(wordId);
|
|
||||||
if (mUpdatingHelper.addNgramEntry(WordIdArrayView::fromArray(prevWordsPtNodePos),
|
|
||||||
wordPtNodePos, bigramProperty, &addedNewEntry)) {
|
|
||||||
if (addedNewEntry) {
|
if (addedNewEntry) {
|
||||||
mBigramCount++;
|
mBigramCount++;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue