Merge "Use IntArrayView to add/remove n-gram entry." into lmp-dev

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp

@@ -231,30 +231,31 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
             &probabilityEntryToWrite);
 }
 
-bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry(
+bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
-        const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam,
+        const BigramProperty *const bigramProperty, bool *const outAddedNewEntry) {
-        const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
+    if (!mBigramPolicy->addNewEntry(prevWordIds[0], wordId, bigramProperty, outAddedNewEntry)) {
-    if (!mBigramPolicy->addNewEntry(sourcePtNodeParams->getTerminalId(),
-            targetPtNodeParam->getTerminalId(), bigramProperty, outAddedNewBigram)) {
         AKLOGE("Cannot add new bigram entry. terminalId: %d, targetTerminalId: %d",
                 sourcePtNodeParams->getTerminalId(), targetPtNodeParam->getTerminalId());
         return false;
     }
-    if (!sourcePtNodeParams->hasBigrams()) {
+    const int ptNodePos =
+            mBuffers->getTerminalPositionLookupTable()->getTerminalPtNodePosition(prevWordIds[0]);
+    const PtNodeParams sourcePtNodeParams =
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
+    if (!sourcePtNodeParams.hasBigrams()) {
         // Update has bigrams flag.
-        return updatePtNodeFlags(sourcePtNodeParams->getHeadPos(),
+        return updatePtNodeFlags(sourcePtNodeParams.getHeadPos(),
-                sourcePtNodeParams->isBlacklisted(), sourcePtNodeParams->isNotAWord(),
+                sourcePtNodeParams.isBlacklisted(), sourcePtNodeParams.isNotAWord(),
-                sourcePtNodeParams->isTerminal(), sourcePtNodeParams->hasShortcutTargets(),
+                sourcePtNodeParams.isTerminal(), sourcePtNodeParams.hasShortcutTargets(),
                 true /* hasBigrams */,
-                sourcePtNodeParams->getCodePointCount() > 1 /* hasMultipleChars */);
+                sourcePtNodeParams.getCodePointCount() > 1 /* hasMultipleChars */);
     }
     return true;
 }
 
-bool Ver4PatriciaTrieNodeWriter::removeBigramEntry(
+bool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWordIds,
-        const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam) {
+        const int wordId) {
-    return mBigramPolicy->removeEntry(sourcePtNodeParams->getTerminalId(),
+    return mBigramPolicy->removeEntry(prevWordIds[0], wordId);
-            targetPtNodeParam->getTerminalId());
 }
 
 bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h

@@ -29,6 +29,7 @@
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
 #include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
+#include "utils/int_array_view.h"
 
 namespace latinime {
 namespace backward {
@@ -61,8 +62,8 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
             const PtNodeArrayReader *const ptNodeArrayReader,
             Ver4BigramListPolicy *const bigramPolicy, Ver4ShortcutListPolicy *const shortcutPolicy)
             : mTrieBuffer(trieBuffer), mBuffers(buffers), mHeaderPolicy(headerPolicy),
-              mReadingHelper(ptNodeReader, ptNodeArrayReader), mBigramPolicy(bigramPolicy),
+              mPtNodeReader(ptNodeReader), mReadingHelper(ptNodeReader, ptNodeArrayReader),
-              mShortcutPolicy(shortcutPolicy) {}
+              mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy) {}
 
     virtual ~Ver4PatriciaTrieNodeWriter() {}
 
@@ -92,12 +93,10 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
             const UnigramProperty *const unigramProperty, int *const ptNodeWritingPos);
 
-    virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
-            const PtNodeParams *const targetPtNodeParam, const BigramProperty *const bigramProperty,
+            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
-            bool *const outAddedNewBigram);
 
-    virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool removeNgramEntry(const WordIdArrayView prevWordIds, const int wordId);
-            const PtNodeParams *const targetPtNodeParam);
 
     virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
             const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);
@@ -135,6 +134,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     BufferWithExtendableBuffer *const mTrieBuffer;
     Ver4DictBuffers *const mBuffers;
     const HeaderPolicy *const mHeaderPolicy;
+    const PtNodeReader *const mPtNodeReader;
     DynamicPtReadingHelper mReadingHelper;
     Ver4BigramListPolicy *const mBigramPolicy;
     Ver4ShortcutListPolicy *const mShortcutPolicy;

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp

@@ -309,8 +309,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
         return false;
     }
     bool addedNewBigram = false;
-    if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty,
+    if (mUpdatingHelper.addNgramEntry(PtNodePosArrayView::fromObject(prevWordsPtNodePos),
-            &addedNewBigram)) {
+            word1Pos, bigramProperty, &addedNewBigram)) {
         if (addedNewBigram) {
             mBigramCount++;
         }
@@ -350,7 +350,8 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
     if (wordPos == NOT_A_DICT_POS) {
         return false;
     }
-    if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) {
+    if (mUpdatingHelper.removeNgramEntry(
+            PtNodePosArrayView::fromObject(prevWordsPtNodePos), wordPos)) {
         mBigramCount--;
         return true;
     } else {

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp

@@ -84,23 +84,39 @@ bool DynamicPtUpdatingHelper::addUnigramWord(
             unigramProperty, &pos);
 }
 
-bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
+bool DynamicPtUpdatingHelper::addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos,
-        const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
+        const int wordPos, const BigramProperty *const bigramProperty,
-    const PtNodeParams sourcePtNodeParams(
+        bool *const outAddedNewEntry) {
-            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word0Pos));
+    if (prevWordsPtNodePos.empty()) {
-    const PtNodeParams targetPtNodeParams(
+        return false;
-            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word1Pos));
+    }
-    return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams,
+    ASSERT(prevWordsPtNodePos.size() <= MAX_PREV_WORD_COUNT_FOR_N_GRAM);
-            bigramProperty, outAddedNewBigram);
+    int prevWordTerminalIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
+    for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
+        prevWordTerminalIds[i] = mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(
+                prevWordsPtNodePos[i]).getTerminalId();
+    }
+    const WordIdArrayView prevWordIds(prevWordTerminalIds, prevWordsPtNodePos.size());
+    const int wordId =
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(wordPos).getTerminalId();
+    return mPtNodeWriter->addNgramEntry(prevWordIds, wordId, bigramProperty, outAddedNewEntry);
 }
 
-// Remove a bigram relation from word0Pos to word1Pos.
+bool DynamicPtUpdatingHelper::removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos,
-bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
+        const int wordPos) {
-    const PtNodeParams sourcePtNodeParams(
+    if (prevWordsPtNodePos.empty()) {
-            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word0Pos));
+        return false;
-    const PtNodeParams targetPtNodeParams(
+    }
-            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word1Pos));
+    ASSERT(prevWordsPtNodePos.size() <= MAX_PREV_WORD_COUNT_FOR_N_GRAM);
-    return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams);
+    int prevWordTerminalIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
+    for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) {
+        prevWordTerminalIds[i] = mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(
+                prevWordsPtNodePos[i]).getTerminalId();
+    }
+    const WordIdArrayView prevWordIds(prevWordTerminalIds, prevWordsPtNodePos.size());
+    const int wordId =
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(wordPos).getTerminalId();
+    return mPtNodeWriter->removeNgramEntry(prevWordIds, wordId);
 }
 
 bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos,

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h

@@ -19,6 +19,7 @@
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
+#include "utils/int_array_view.h"
 
 namespace latinime {
 
@@ -42,12 +43,12 @@ class DynamicPtUpdatingHelper {
             const int *const wordCodePoints, const int codePointCount,
             const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram);
 
-    // Add a bigram relation from word0Pos to word1Pos.
+    // Add an n-gram entry.
-    bool addBigramWords(const int word0Pos, const int word1Pos,
+    bool addNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos,
-            const BigramProperty *const bigramProperty, bool *const outAddedNewBigram);
+            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
 
-    // Remove a bigram relation from word0Pos to word1Pos.
+    // Remove an n-gram entry.
-    bool removeBigramWords(const int word0Pos, const int word1Pos);
+    bool removeNgramEntry(const PtNodePosArrayView prevWordsPtNodePos, const int wordPos);
 
     // Add a shortcut target.
     bool addShortcutTarget(const int wordPos, const int *const targetCodePoints,

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h

@@ -21,6 +21,7 @@
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
+#include "utils/int_array_view.h"
 
 namespace latinime {
 
@@ -70,12 +71,10 @@ class PtNodeWriter {
     virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
             const UnigramProperty *const unigramProperty, int *const ptNodeWritingPos) = 0;
 
-    virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
-            const PtNodeParams *const targetPtNodeParam, const BigramProperty *const bigramProperty,
+            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry) = 0;
-            bool *const outAddedNewBigram) = 0;
 
-    virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool removeNgramEntry(const WordIdArrayView prevWordIds, const int wordId) = 0;
-            const PtNodeParams *const targetPtNodeParam) = 0;
 
     virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
             const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) = 0;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp

@@ -222,22 +222,19 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition(
             terminalId, &probabilityEntryToWrite);
 }
 
-bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry(
+bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
-        const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam,
         const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
-    if (!mBigramPolicy->addNewEntry(sourcePtNodeParams->getTerminalId(),
+    if (!mBigramPolicy->addNewEntry(prevWordIds[0], wordId, bigramProperty, outAddedNewBigram)) {
-            targetPtNodeParam->getTerminalId(), bigramProperty, outAddedNewBigram)) {
         AKLOGE("Cannot add new bigram entry. terminalId: %d, targetTerminalId: %d",
-                sourcePtNodeParams->getTerminalId(), targetPtNodeParam->getTerminalId());
+                prevWordIds[0], wordId);
         return false;
     }
     return true;
 }
 
-bool Ver4PatriciaTrieNodeWriter::removeBigramEntry(
+bool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWordIds,
-        const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam) {
+        const int wordId) {
-    return mBigramPolicy->removeEntry(sourcePtNodeParams->getTerminalId(),
+    return mBigramPolicy->removeEntry(prevWordIds[0], wordId);
-            targetPtNodeParam->getTerminalId());
 }
 
 bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h

@@ -75,12 +75,10 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     virtual bool writeNewTerminalPtNodeAndAdvancePosition(const PtNodeParams *const ptNodeParams,
             const UnigramProperty *const unigramProperty, int *const ptNodeWritingPos);
 
-    virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool addNgramEntry(const WordIdArrayView prevWordIds, const int wordId,
-            const PtNodeParams *const targetPtNodeParam, const BigramProperty *const bigramProperty,
+            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
-            bool *const outAddedNewBigram);
 
-    virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams,
+    virtual bool removeNgramEntry(const WordIdArrayView prevWordIds, const int wordId);
-            const PtNodeParams *const targetPtNodeParam);
 
     virtual bool updateAllBigramEntriesAndDeleteUselessEntries(
             const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount);

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp

@@ -292,6 +292,7 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
     int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
     prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
             false /* tryLowerCaseSearch */);
+    const auto prevWordsPtNodePosView = PtNodePosArrayView::fromFixedSizeArray(prevWordsPtNodePos);
     // TODO: Support N-gram.
     if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
         if (prevWordsInfo->isNthPrevWordBeginningOfSentence(1 /* n */)) {
@@ -319,10 +320,10 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
     if (word1Pos == NOT_A_DICT_POS) {
         return false;
     }
-    bool addedNewBigram = false;
+    bool addedNewEntry = false;
-    if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty,
+    if (mUpdatingHelper.addNgramEntry(prevWordsPtNodePosView, word1Pos, bigramProperty,
-            &addedNewBigram)) {
+            &addedNewEntry)) {
-        if (addedNewBigram) {
+        if (addedNewEntry) {
             mBigramCount++;
         }
         return true;
@@ -352,6 +353,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
     int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
     prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
             false /* tryLowerCaseSerch */);
+    const auto prevWordsPtNodePosView = PtNodePosArrayView::fromFixedSizeArray(prevWordsPtNodePos);
     // TODO: Support N-gram.
     if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
         return false;
@@ -361,7 +363,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
     if (wordPos == NOT_A_DICT_POS) {
         return false;
     }
-    if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) {
+    if (mUpdatingHelper.removeNgramEntry(prevWordsPtNodePosView, wordPos)) {
         mBigramCount--;
         return true;
     } else {

native/jni/src/utils/int_array_view.h

@@ -56,6 +56,16 @@ class IntArrayView {
     explicit IntArrayView(const std::vector<int> &vector)
             : mPtr(vector.data()), mSize(vector.size()) {}
 
+    template <int N>
+    AK_FORCE_INLINE static IntArrayView fromFixedSizeArray(const int (&array)[N]) {
+        return IntArrayView(array, N);
+    }
+
+    // Returns a view that points one int object. Does not take ownership of the given object.
+    AK_FORCE_INLINE static IntArrayView fromObject(const int *const object) {
+        return IntArrayView(object, 1);
+    }
+
     AK_FORCE_INLINE int operator[](const size_t index) const {
         ASSERT(index < mSize);
         return mPtr[index];
@@ -89,6 +99,7 @@ class IntArrayView {
 };
 
 using WordIdArrayView = IntArrayView;
+using PtNodePosArrayView = IntArrayView;
 
 } // namespace latinime
 #endif // LATINIME_MEMORY_VIEW_H

native/jni/tests/utils/int_array_view_test.cpp

@@ -43,5 +43,19 @@ TEST(IntArrayViewTest, TestIteration) {
     EXPECT_EQ(expectedIndex, intArrayView.size());
 }
 
+TEST(IntArrayViewTest, TestConstructFromArray) {
+    const size_t ARRAY_SIZE = 100;
+    int intArray[ARRAY_SIZE];
+    const auto intArrayView = IntArrayView::fromFixedSizeArray(intArray);
+    EXPECT_EQ(ARRAY_SIZE, intArrayView.size());
+}
+
+TEST(IntArrayViewTest, TestConstructFromObject) {
+    const int object = 10;
+    const auto intArrayView = IntArrayView::fromObject(&object);
+    EXPECT_EQ(1, intArrayView.size());
+    EXPECT_EQ(object, intArrayView[0]);
+}
+
 }  // namespace
 }  // namespace latinime