nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Support adding n-gram entry in structure policy.

Browse source 
Bug: 14425059
Change-Id: Ia207d3c4735b1d6d43f18d18a70d28af613cb458
This commit is contained in:
Keisuke Kuroyanagi 2014-09-16 15:37:09 +09:00
parent 7eb6e28b9e
commit 0c0b8207cd
3 changed files with 7 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h
View file

@ -43,10 +43,12 @@ class DynamicPtUpdatingHelper {
const int *const wordCodePoints, const int codePointCount, const int *const wordCodePoints, const int codePointCount,
const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram); const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram);
// TODO: Remove after stopping supporting v402.
// Add an n-gram entry. // Add an n-gram entry.
bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos, bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos,
const BigramProperty *const bigramProperty, bool *const outAddedNewEntry); const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
// TODO: Remove after stopping supporting v402.
// Remove an n-gram entry. // Remove an n-gram entry.
bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos); bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos);

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
View file

@ -211,19 +211,17 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId, bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) { const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
// TODO: Support n-gram.
LanguageModelDictContent *const languageModelDictContent = LanguageModelDictContent *const languageModelDictContent =
mBuffers->getMutableLanguageModelDictContent(); mBuffers->getMutableLanguageModelDictContent();
const ProbabilityEntry probabilityEntry = const ProbabilityEntry probabilityEntry =
languageModelDictContent->getNgramProbabilityEntry( languageModelDictContent->getNgramProbabilityEntry(prevWordIds, wordId);
prevWordIds.limit(1 /* maxSize */), wordId);
const ProbabilityEntry probabilityEntryOfBigramProperty(bigramProperty); const ProbabilityEntry probabilityEntryOfBigramProperty(bigramProperty);
const ProbabilityEntry updatedProbabilityEntry = createUpdatedEntryFrom( const ProbabilityEntry updatedProbabilityEntry = createUpdatedEntryFrom(
&probabilityEntry, &probabilityEntryOfBigramProperty); &probabilityEntry, &probabilityEntryOfBigramProperty);
if (!languageModelDictContent->setNgramProbabilityEntry( if (!languageModelDictContent->setNgramProbabilityEntry(
prevWordIds.limit(1 /* maxSize */), wordId, &updatedProbabilityEntry)) { prevWordIds, wordId, &updatedProbabilityEntry)) {
AKLOGE("Cannot add new ngram entry. prevWordId: %d, wordId: %d", AKLOGE("Cannot add new ngram entry. prevWordId[0]: %d, prevWordId.size(): %zd, wordId: %d",
prevWordIds[0], wordId); prevWordIds[0], prevWordIds.size(), wordId);
return false; return false;
} }
if (!probabilityEntry.isValid() && outAddedNewBigram) { if (!probabilityEntry.isValid() && outAddedNewBigram) {

11
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View file

@ -335,17 +335,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
if (wordId == NOT_A_WORD_ID) { if (wordId == NOT_A_WORD_ID) {
return false; return false;
} }
// TODO: Support N-gram.
bool addedNewEntry = false; bool addedNewEntry = false;
WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordsPtNodePos; if (mNodeWriter.addNgramEntry(prevWordIds, wordId, bigramProperty, &addedNewEntry)) {
for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
prevWordsPtNodePos[i] = mBuffers->getTerminalPositionLookupTable()
->getTerminalPtNodePosition(prevWordIds[i]);
}
const int wordPtNodePos = mBuffers->getTerminalPositionLookupTable()
->getTerminalPtNodePosition(wordId);
if (mUpdatingHelper.addNgramEntry(WordIdArrayView::fromArray(prevWordsPtNodePos),
wordPtNodePos, bigramProperty, &addedNewEntry)) {
if (addedNewEntry) { if (addedNewEntry) {
mBigramCount++; mBigramCount++;
} }