Merge "Implement Ver4PatriciaTriePolicy::needsToRunGC()."

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h

@@ -46,6 +46,10 @@ class SingleDictContent : public DictContent {
         return mIsValid;
     }
 
+    bool isNearSizeLimit() const {
+        return mExpandableContentBuffer.isNearSizeLimit();
+    }
+
  protected:
     BufferWithExtendableBuffer *getWritableBuffer() {
         return &mExpandableContentBuffer;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h

@@ -69,6 +69,12 @@ class SparseTableDictContent : public DictContent {
         return mIsValid;
     }
 
+    bool isNearSizeLimit() const {
+        return mExpandableLookupTableBuffer.isNearSizeLimit()
+                || mExpandableAddressTableBuffer.isNearSizeLimit()
+                || mExpandableContentBuffer.isNearSizeLimit();
+    }
+
  protected:
     SparseTable *getUpdatableAddressLookupTable() {
         return &mAddressLookupTable;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h

@@ -49,6 +49,14 @@ class Ver4DictBuffers {
                 && mShortcutDictContent.isValid();
     }
 
+    AK_FORCE_INLINE bool isNearSizeLimit() const {
+        return mExpandableTrieBuffer.isNearSizeLimit()
+                || mTerminalPositionLookupTable.isNearSizeLimit()
+                || mProbabilityDictContent.isNearSizeLimit()
+                || mBigramDictContent.isNearSizeLimit()
+                || mShortcutDictContent.isNearSizeLimit();
+    }
+
     AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() {
         return &mExpandableHeaderBuffer;
     }

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp

@@ -33,6 +33,9 @@ const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
 // Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
 // TODO: Make MAX_DICTIONARY_SIZE 8MB.
 const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
+// Extended region size, which is not GCed region size in dict file + additional buffer size, is
+// limited to 1MB to prevent from inefficient traversing.
+const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
 
 const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
 const int Ver4DictConstants::PROBABILITY_SIZE = 1;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h

@@ -35,6 +35,7 @@ class Ver4DictConstants {
     static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
 
     static const int MAX_DICTIONARY_SIZE;
+    static const int MAX_DICT_EXTENDED_REGION_SIZE;
 
     static const int NOT_A_TERMINAL_ID;
     static const int PROBABILITY_SIZE;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp

@@ -19,7 +19,6 @@
 #include "suggest/core/dicnode/dic_node.h"
 #include "suggest/core/dicnode/dic_node_vector.h"
 #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
-#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 #include "suggest/policyimpl/dictionary/utils/probability_utils.h"
@@ -228,7 +227,25 @@ void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
 }
 
 bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
-    // TODO: Implement.
+    if (!mBuffers.get()->isUpdatable()) {
+        AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
+        return false;
+    }
+    if (mBuffers.get()->isNearSizeLimit()) {
+        // Additional buffer size is near the limit.
+        return true;
+    } else if (mHeaderPolicy.getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
+            > Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE) {
+        // Total extended region size of the trie exceeds the limit.
+        return true;
+    } else if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS
+            && mDictBuffer->getUsedAdditionalBufferSize() > 0) {
+        // Needs to reduce dictionary size.
+        return true;
+    } else if (mHeaderPolicy.isDecayingDict()) {
+        return mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay(
+                mindsBlockByGC, mUnigramCount, mBigramCount, &mHeaderPolicy);
+    }
     return false;
 }
 

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h

@@ -53,7 +53,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
                       mHeaderPolicy.isDecayingDict()),
               mWritingHelper(mBuffers.get()),
               mUnigramCount(mHeaderPolicy.getUnigramCount()),
-              mBigramCount(mHeaderPolicy.getBigramCount()) {};
+              mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {};
 
     AK_FORCE_INLINE int getRootPosition() const {
         return 0;
@@ -125,6 +125,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     Ver4PatriciaTrieWritingHelper mWritingHelper;
     int mUnigramCount;
     int mBigramCount;
+    bool mNeedsToDecayForTesting;
 };
 } // namespace latinime
 #endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H