Add willBecomeNonTerminal flag.

willBecomeNonTerminal flag is used during GC to indicate
the PtNode is no longer terminal, even though the PtNode has
terminal attributes.

Bug: 11073222
Change-Id: I2514bc4106fa7539201979affcfb713d2b8430a3

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h

@@ -114,6 +114,10 @@ class PtNodeParams {
         return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags);
     }
 
+    AK_FORCE_INLINE bool willBecomeNonTerminal() const {
+        return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags);
+    }
+
     AK_FORCE_INLINE bool hasChildren() const {
         return mChildrenPos != NOT_A_DICT_POS;
     }

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h

@@ -48,6 +48,9 @@ class PtNodeWriter {
     virtual bool markPtNodeAsMoved(const PtNodeParams *const toBeUpdatedPtNodeParams,
             const int movedPos, const int bigramLinkedNodePos) = 0;
 
+    virtual bool markPtNodeAsWillBecomeNonTerminal(
+            const PtNodeParams *const toBeUpdatedPtNodeParams) = 0;
+
     virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams,
             const int probability, const int timestamp) = 0;
 

native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp

@@ -42,6 +42,10 @@ bool DynamicPatriciaTrieGcEventListeners
         }
         if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) {
             isUselessPtNode = true;
+            if (!mPtNodeWriter->markPtNodeAsWillBecomeNonTerminal(ptNodeParams)) {
+                AKLOGE("Cannot mark PtNode as willBecomeNonTerminal.");
+                return false;
+            }
         }
     }
     if (mChildrenValue > 0) {

native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp

@@ -27,6 +27,7 @@ const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0;
 const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0;
 const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40;
 const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80;
+const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00;
 
 // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0.
 // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum

native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h

@@ -54,12 +54,19 @@ class DynamicPatriciaTrieReadingUtils {
         return FLAG_IS_DELETED == (MASK_MOVED & flags);
     }
 
+    static AK_FORCE_INLINE bool willBecomeNonTerminal(const NodeFlags flags) {
+        return FLAG_WILL_BECOME_NON_TERMINAL == (MASK_MOVED & flags);
+    }
+
     static AK_FORCE_INLINE NodeFlags updateAndGetFlags(const NodeFlags originalFlags,
-            const bool isMoved, const bool isDeleted) {
+            const bool isMoved, const bool isDeleted, const bool willBecomeNonTerminal) {
         NodeFlags flags = originalFlags;
+        flags = willBecomeNonTerminal ?
+                ((flags & (~MASK_MOVED)) | FLAG_WILL_BECOME_NON_TERMINAL) : flags;
         flags = isMoved ? ((flags & (~MASK_MOVED)) | FLAG_IS_MOVED) : flags;
         flags = isDeleted ? ((flags & (~MASK_MOVED)) | FLAG_IS_DELETED) : flags;
-        flags = (!isMoved && !isDeleted) ? ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags;
+        flags = (!isMoved && !isDeleted && !willBecomeNonTerminal) ?
+                ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags;
         return flags;
     }
 
@@ -70,6 +77,7 @@ class DynamicPatriciaTrieReadingUtils {
     static const NodeFlags FLAG_IS_NOT_MOVED;
     static const NodeFlags FLAG_IS_MOVED;
     static const NodeFlags FLAG_IS_DELETED;
+    static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL;
 };
 } // namespace latinime
 #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp

@@ -44,7 +44,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted(
             PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
     const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
             DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
-                    true /* isDeleted */);
+                    true /* isDeleted */, false /* willBecomeNonTerminal */);
     int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
     // Update flags.
     if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
@@ -74,7 +74,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
             PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
     const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
             DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */,
-                    false /* isDeleted */);
+                    false /* isDeleted */,  false /* willBecomeNonTerminal */);
     int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
     // Update flags.
     if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
@@ -106,6 +106,26 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
     return true;
 }
 
+bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal(
+        const PtNodeParams *const toBeUpdatedPtNodeParams) {
+    int pos = toBeUpdatedPtNodeParams->getHeadPos();
+    const bool usesAdditionalBuffer = mTrieBuffer->isInAdditionalBuffer(pos);
+    const uint8_t *const dictBuf = mTrieBuffer->getBuffer(usesAdditionalBuffer);
+    if (usesAdditionalBuffer) {
+        pos -= mTrieBuffer->getOriginalBufferSize();
+    }
+    // Read original flags
+    const PatriciaTrieReadingUtils::NodeFlags originalFlags =
+            PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
+    const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
+            DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
+                    false /* isDeleted */, true /* willBecomeNonTerminal */);
+    int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
+    // Update flags.
+    return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
+            &writingPos);
+}
+
 bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbability(
         const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability,
         const int timestamp) {
@@ -281,12 +301,14 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition(
         return false;
     }
     int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID;
-    if (ptNodeParams->getTerminalId() != Ver4DictConstants::NOT_A_TERMINAL_ID) {
+    if (!ptNodeParams->willBecomeNonTerminal()) {
-        terminalId = ptNodeParams->getTerminalId();
+        if (ptNodeParams->getTerminalId() != Ver4DictConstants::NOT_A_TERMINAL_ID) {
-    } else if (ptNodeParams->isTerminal()) {
+            terminalId = ptNodeParams->getTerminalId();
-        // Write terminal information using a new terminal id.
+        } else if (ptNodeParams->isTerminal()) {
-        // Get a new unused terminal id.
+            // Write terminal information using a new terminal id.
-        terminalId = mBuffers->getTerminalPositionLookupTable()->getNextTerminalId();
+            // Get a new unused terminal id.
+            terminalId = mBuffers->getTerminalPositionLookupTable()->getNextTerminalId();
+        }
     }
     const int isTerminal = terminalId != Ver4DictConstants::NOT_A_TERMINAL_ID;
     if (isTerminal) {

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h

@@ -54,6 +54,9 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     virtual bool markPtNodeAsMoved(const PtNodeParams *const toBeUpdatedPtNodeParams,
             const int movedPos, const int bigramLinkedNodePos);
 
+    virtual bool markPtNodeAsWillBecomeNonTerminal(
+            const PtNodeParams *const toBeUpdatedPtNodeParams);
+
     virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams,
             const int newProbability, const int timestamp);