From 88af9eb862bf12b6d72f3d595708bd530ea03bfa Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Mon, 9 Dec 2013 17:59:15 +0900 Subject: [PATCH] Add willBecomeNonTerminal flag. willBecomeNonTerminal flag is used during GC to indicate the PtNode is no longer terminal, even though the PtNode has terminal attributes. Bug: 11073222 Change-Id: I2514bc4106fa7539201979affcfb713d2b8430a3 --- .../structure/pt_common/pt_node_params.h | 4 ++ .../structure/pt_common/pt_node_writer.h | 3 ++ ...namic_patricia_trie_gc_event_listeners.cpp | 4 ++ .../dynamic_patricia_trie_reading_utils.cpp | 1 + .../v3/dynamic_patricia_trie_reading_utils.h | 12 +++++- .../v4/ver4_patricia_trie_node_writer.cpp | 38 +++++++++++++++---- .../v4/ver4_patricia_trie_node_writer.h | 3 ++ 7 files changed, 55 insertions(+), 10 deletions(-) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h index e99bb216a..2a43e39bf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h @@ -114,6 +114,10 @@ class PtNodeParams { return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags); } + AK_FORCE_INLINE bool willBecomeNonTerminal() const { + return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags); + } + AK_FORCE_INLINE bool hasChildren() const { return mChildrenPos != NOT_A_DICT_POS; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h index c7a36e796..5c6a36540 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h @@ -48,6 +48,9 @@ class PtNodeWriter { virtual bool markPtNodeAsMoved(const PtNodeParams *const toBeUpdatedPtNodeParams, const int movedPos, const int bigramLinkedNodePos) = 0; + virtual bool markPtNodeAsWillBecomeNonTerminal( + const PtNodeParams *const toBeUpdatedPtNodeParams) = 0; + virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams, const int probability, const int timestamp) = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp index 22d4f7ac8..a06962eb0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp @@ -42,6 +42,10 @@ bool DynamicPatriciaTrieGcEventListeners } if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) { isUselessPtNode = true; + if (!mPtNodeWriter->markPtNodeAsWillBecomeNonTerminal(ptNodeParams)) { + AKLOGE("Cannot mark PtNode as willBecomeNonTerminal."); + return false; + } } } if (mChildrenValue > 0) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp index e94925365..6cd127487 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp @@ -27,6 +27,7 @@ const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; +const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h index 67c3cc57e..25b0333a5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h @@ -54,12 +54,19 @@ class DynamicPatriciaTrieReadingUtils { return FLAG_IS_DELETED == (MASK_MOVED & flags); } + static AK_FORCE_INLINE bool willBecomeNonTerminal(const NodeFlags flags) { + return FLAG_WILL_BECOME_NON_TERMINAL == (MASK_MOVED & flags); + } + static AK_FORCE_INLINE NodeFlags updateAndGetFlags(const NodeFlags originalFlags, - const bool isMoved, const bool isDeleted) { + const bool isMoved, const bool isDeleted, const bool willBecomeNonTerminal) { NodeFlags flags = originalFlags; + flags = willBecomeNonTerminal ? + ((flags & (~MASK_MOVED)) | FLAG_WILL_BECOME_NON_TERMINAL) : flags; flags = isMoved ? ((flags & (~MASK_MOVED)) | FLAG_IS_MOVED) : flags; flags = isDeleted ? ((flags & (~MASK_MOVED)) | FLAG_IS_DELETED) : flags; - flags = (!isMoved && !isDeleted) ? ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; + flags = (!isMoved && !isDeleted && !willBecomeNonTerminal) ? + ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; return flags; } @@ -70,6 +77,7 @@ class DynamicPatriciaTrieReadingUtils { static const NodeFlags FLAG_IS_NOT_MOVED; static const NodeFlags FLAG_IS_MOVED; static const NodeFlags FLAG_IS_DELETED; + static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL; }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index d52cc4925..2a9acf5dc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -44,7 +44,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted( PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, - true /* isDeleted */); + true /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, @@ -74,7 +74,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, - false /* isDeleted */); + false /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, @@ -106,6 +106,26 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( return true; } +bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( + const PtNodeParams *const toBeUpdatedPtNodeParams) { + int pos = toBeUpdatedPtNodeParams->getHeadPos(); + const bool usesAdditionalBuffer = mTrieBuffer->isInAdditionalBuffer(pos); + const uint8_t *const dictBuf = mTrieBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + pos -= mTrieBuffer->getOriginalBufferSize(); + } + // Read original flags + const PatriciaTrieReadingUtils::NodeFlags originalFlags = + PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); + const PatriciaTrieReadingUtils::NodeFlags updatedFlags = + DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + false /* isDeleted */, true /* willBecomeNonTerminal */); + int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); + // Update flags. + return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + &writingPos); +} + bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbability( const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability, const int timestamp) { @@ -281,12 +301,14 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( return false; } int terminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; - if (ptNodeParams->getTerminalId() != Ver4DictConstants::NOT_A_TERMINAL_ID) { - terminalId = ptNodeParams->getTerminalId(); - } else if (ptNodeParams->isTerminal()) { - // Write terminal information using a new terminal id. - // Get a new unused terminal id. - terminalId = mBuffers->getTerminalPositionLookupTable()->getNextTerminalId(); + if (!ptNodeParams->willBecomeNonTerminal()) { + if (ptNodeParams->getTerminalId() != Ver4DictConstants::NOT_A_TERMINAL_ID) { + terminalId = ptNodeParams->getTerminalId(); + } else if (ptNodeParams->isTerminal()) { + // Write terminal information using a new terminal id. + // Get a new unused terminal id. + terminalId = mBuffers->getTerminalPositionLookupTable()->getNextTerminalId(); + } } const int isTerminal = terminalId != Ver4DictConstants::NOT_A_TERMINAL_ID; if (isTerminal) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 73b2ae309..cc66688f7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -54,6 +54,9 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { virtual bool markPtNodeAsMoved(const PtNodeParams *const toBeUpdatedPtNodeParams, const int movedPos, const int bigramLinkedNodePos); + virtual bool markPtNodeAsWillBecomeNonTerminal( + const PtNodeParams *const toBeUpdatedPtNodeParams); + virtual bool updatePtNodeProbability(const PtNodeParams *const toBeUpdatedPtNodeParams, const int newProbability, const int timestamp);