From 90cff563a0fa672bc3a72be22b1dfbd7b831b34e Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Thu, 12 Dec 2013 14:53:11 +0900 Subject: [PATCH] Remove or rename native codes for ver3 dictionary format. Change-Id: I2d1fe515fc8bbbc7095581fccb4af94377ec11b2 --- native/jni/Android.mk | 12 ++--- .../bigram/bigram_list_read_write_utils.cpp | 6 --- .../dictionary/header/header_policy.h | 2 - .../header/header_read_write_utils.cpp | 3 -- .../dynamic_pt_gc_event_listeners.cpp} | 22 ++++----- .../dynamic_pt_gc_event_listeners.h} | 21 ++++----- .../dynamic_pt_reading_helper.cpp} | 26 +++++----- .../dynamic_pt_reading_helper.h} | 15 +++--- .../dynamic_pt_reading_utils.cpp} | 27 +++++------ .../dynamic_pt_reading_utils.h} | 10 ++-- .../dynamic_pt_updating_helper.cpp} | 42 ++++++++--------- .../dynamic_pt_updating_helper.h} | 17 ++++--- .../dynamic_pt_writing_utils.cpp} | 47 +++++++++---------- .../dynamic_pt_writing_utils.h} | 16 +++---- .../structure/pt_common/pt_node_params.h | 6 +-- .../v2/patricia_trie_reading_utils.h | 1 + .../structure/v4/ver4_dict_constants.h | 1 + .../v4/ver4_patricia_trie_node_reader.cpp | 10 ++-- .../v4/ver4_patricia_trie_node_writer.cpp | 38 +++++++-------- .../v4/ver4_patricia_trie_node_writer.h | 4 +- .../v4/ver4_patricia_trie_policy.cpp | 10 ++-- .../structure/v4/ver4_patricia_trie_policy.h | 4 +- .../v4/ver4_patricia_trie_writing_helper.cpp | 12 ++--- .../v4/ver4_patricia_trie_writing_helper.h | 4 +- .../utils/dict_file_writing_utils.cpp | 4 +- .../dictionary/utils/format_utils.cpp | 2 - .../dictionary/utils/format_utils.h | 1 - 27 files changed, 173 insertions(+), 190 deletions(-) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_gc_event_listeners.cpp => pt_common/dynamic_pt_gc_event_listeners.cpp} (84%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_gc_event_listeners.h => pt_common/dynamic_pt_gc_event_listeners.h} (88%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_reading_helper.cpp => pt_common/dynamic_pt_reading_helper.cpp} (92%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_reading_helper.h => pt_common/dynamic_pt_reading_helper.h} (96%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_reading_utils.cpp => pt_common/dynamic_pt_reading_utils.cpp} (64%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_reading_utils.h => pt_common/dynamic_pt_reading_utils.h} (90%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_updating_helper.cpp => pt_common/dynamic_pt_updating_helper.cpp} (88%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_updating_helper.h => pt_common/dynamic_pt_updating_helper.h} (89%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_writing_utils.cpp => pt_common/dynamic_pt_writing_utils.cpp} (68%) rename native/jni/src/suggest/policyimpl/dictionary/structure/{v3/dynamic_patricia_trie_writing_utils.h => pt_common/dynamic_pt_writing_utils.h} (84%) diff --git a/native/jni/Android.mk b/native/jni/Android.mk index f2c6d3bec..e11e706f3 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -76,15 +76,15 @@ LATIN_IME_CORE_SRC_FILES := \ $(addprefix suggest/policyimpl/dictionary/bigram/, \ bigram_list_read_write_utils.cpp \ ver4_bigram_list_policy.cpp) \ + $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \ + dynamic_pt_gc_event_listeners.cpp \ + dynamic_pt_reading_helper.cpp \ + dynamic_pt_reading_utils.cpp \ + dynamic_pt_updating_helper.cpp \ + dynamic_pt_writing_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v2/, \ patricia_trie_policy.cpp \ patricia_trie_reading_utils.cpp) \ - $(addprefix suggest/policyimpl/dictionary/structure/v3/, \ - dynamic_patricia_trie_gc_event_listeners.cpp \ - dynamic_patricia_trie_reading_helper.cpp \ - dynamic_patricia_trie_reading_utils.cpp \ - dynamic_patricia_trie_updating_helper.cpp \ - dynamic_patricia_trie_writing_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/, \ ver4_dict_buffers.cpp \ ver4_dict_constants.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index 0ef6ccf37..7d0d09631 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -16,7 +16,6 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -78,11 +77,6 @@ const BigramListReadWriteUtils::BigramFlags offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); break; } - if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) { - return NOT_A_DICT_POS; - } else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) { - return origin; - } if (isOffsetNegative(flags)) { return origin - offset; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 5e8b5f11e..1208d2c2a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -87,8 +87,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { switch (mDictFormatVersion) { case FormatUtils::VERSION_2: return FormatUtils::VERSION_2; - case FormatUtils::VERSION_3: - return FormatUtils::VERSION_3; case FormatUtils::VERSION_4: return FormatUtils::VERSION_4; default: diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index 2d4547066..6b4598642 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -89,9 +89,6 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; case FormatUtils::VERSION_2: // Version 2 dictionary writing is not supported. return false; - case FormatUtils::VERSION_3: - return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_3 /* data */, - HEADER_DICTIONARY_VERSION_SIZE, writingPos); case FormatUtils::VERSION_4: return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */, HEADER_DICTIONARY_VERSION_SIZE, writingPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp similarity index 84% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp index bcba67035..8f42df6d2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" namespace latinime { -bool DynamicPatriciaTrieGcEventListeners +bool DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless @@ -63,7 +63,7 @@ bool DynamicPatriciaTrieGcEventListeners return true; } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability +bool DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) { int bigramEntryCount = 0; @@ -77,7 +77,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit } // Writes dummy PtNode array size when the head of PtNode array is read. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onDescend(const int ptNodeArrayPos) { mValidPtNodeCount = 0; int writingPos = mBufferToWrite->getTailPosition(); @@ -86,21 +86,21 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. mPtNodeArraySizeFieldPos = writingPos; - return DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + return DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, 0 /* arraySize */, &writingPos); } // Write PtNode array terminal and actual PtNode array size. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onReadingPtNodeArrayTail() { int writingPos = mBufferToWrite->getTailPosition(); // Write PtNode array terminal. - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } // Write actual PtNode array size. - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) { return false; } @@ -108,7 +108,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo } // Write valid PtNode to buffer and memorize mapping from the old position to the new position. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (ptNodeParams->isDeleted()) { // Current PtNode is not written in new buffer because it has been deleted. @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos); } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields +bool DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // Updates parent position. int bigramCount = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h similarity index 88% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h index 562eb69bd..d8867754d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h @@ -14,14 +14,14 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H +#ifndef LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H +#define LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H #include #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "utils/hash_map_compat.h" @@ -29,14 +29,13 @@ namespace latinime { class PtNodeParams; -// TODO: Move to pt_common. -class DynamicPatriciaTrieGcEventListeners { +class DynamicPtGcEventListeners { public: // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC. // TODO: Concatenate non-terminal PtNodes. class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( PtNodeWriter *const ptNodeWriter) @@ -81,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners { // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram // entries. class TraversePolicyToUpdateBigramProbability - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter) : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {} @@ -106,7 +105,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, @@ -134,7 +133,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToUpdateAllPositionFields - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter, const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) @@ -168,7 +167,7 @@ class DynamicPatriciaTrieGcEventListeners { }; private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtGcEventListeners); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H */ +#endif /* LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp similarity index 92% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp index b3fdbeb78..b918e0765 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp @@ -14,25 +14,25 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "utils/char_utils.h" namespace latinime { // To avoid infinite loop caused by invalid or malicious forward links. -const int DynamicPatriciaTrieReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const int DynamicPatriciaTrieReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const size_t DynamicPatriciaTrieReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; +const int DynamicPtReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const int DynamicPtReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const size_t DynamicPtReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; // Visits all PtNodes in post-order depth first manner. // For example, visits c -> b -> y -> x -> a for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedChildren = false; // Descend from the root to the root PtNode array. @@ -92,7 +92,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa // For example, visits a -> b -> x -> c -> y for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedAllPtNodesInArray = false; bool alreadyVisitedChildren = false; @@ -169,7 +169,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor return !isError(); } -int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( +int DynamicPtReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) { // This method traverses parent nodes from the terminal by following parent pointers; thus, // node code points are stored in the buffer in the reverse order. @@ -211,7 +211,7 @@ int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePo return totalCodePointCount; } -int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, +int DynamicPtReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) { int searchCodePoints[length]; for (int i = 0; i < length; ++i) { @@ -257,7 +257,7 @@ int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int // Read node array size and process empty node arrays. Nodes and arrays are counted up in this // method to avoid an infinite loop. -void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { +void DynamicPtReadingHelper::nextPtNodeArray() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of a bug or a broken dictionary. AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d", @@ -308,7 +308,7 @@ void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { } // Follow the forward link and read the next node array if exists. -void DynamicPatriciaTrieReadingHelper::followForwardLink() { +void DynamicPtReadingHelper::followForwardLink() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of bug or broken dictionary. AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d", @@ -324,12 +324,12 @@ void DynamicPatriciaTrieReadingHelper::followForwardLink() { mReadingState.mPos -= mBuffer->getOriginalBufferSize(); } const int forwardLinkPosition = - DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); + DynamicPtReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); if (usesAdditionalBuffer) { mReadingState.mPos += mBuffer->getOriginalBufferSize(); } mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos; - if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { + if (DynamicPtReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { // Follow the forward link. mReadingState.mPos += forwardLinkPosition; nextPtNodeArray(); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h similarity index 96% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h index 197027313..a69490943 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_READING_HELPER_H +#define LATINIME_DYNAMIC_PT_READING_HELPER_H #include #include @@ -34,8 +34,7 @@ class DictionaryShortcutsStructurePolicy; * This class is used for traversing dynamic patricia trie. This class supports iterating nodes and * dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop. */ -// TODO: Move to pt_common. -class DynamicPatriciaTrieReadingHelper { +class DynamicPtReadingHelper { public: class TraversingEventListener { public: @@ -60,12 +59,12 @@ class DynamicPatriciaTrieReadingHelper { DISALLOW_COPY_AND_ASSIGN(TraversingEventListener); }; - DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer, + DynamicPtReadingHelper(const BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader) : mIsError(false), mReadingState(), mBuffer(buffer), mPtNodeReader(ptNodeReader), mReadingStateStack() {} - ~DynamicPatriciaTrieReadingHelper() {} + ~DynamicPtReadingHelper() {} AK_FORCE_INLINE bool isError() const { return mIsError; @@ -205,7 +204,7 @@ class DynamicPatriciaTrieReadingHelper { const bool forceLowerCaseSearch); private: - DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieReadingHelper); + DISALLOW_COPY_AND_ASSIGN(DynamicPtReadingHelper); // This class encapsulates the reading state of a position in the dictionary. It points at a // specific PtNode in the dictionary. @@ -267,4 +266,4 @@ class DynamicPatriciaTrieReadingHelper { } }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp similarity index 64% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp index 6cd127487..3586b50ab 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp @@ -14,39 +14,38 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { -typedef DynamicPatriciaTrieReadingUtils DptReadingUtils; - -const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::MASK_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_MOVED = 0x40; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_DELETED = 0x80; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum // value of offsets, which is 0x7FFFFF is used to represent 0 offset. -const int DptReadingUtils::DICT_OFFSET_INVALID = 0; -const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; +const int DynamicPtReadingUtils::DICT_OFFSET_INVALID = 0; +const int DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; -/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, +/* static */ int DynamicPtReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } -/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } -/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) { +/* static */ int DynamicPtReadingUtils::getParentPtNodePos(const int parentOffset, + const int ptNodePos) { if (parentOffset == DICT_OFFSET_INVALID) { return NOT_A_DICT_POS; } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) { @@ -56,7 +55,7 @@ const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; } } -/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h similarity index 90% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h index 25b0333a5..89ae12c0b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_READING_UTILS_H +#define LATINIME_DYNAMIC_PT_READING_UTILS_H #include @@ -23,7 +23,7 @@ namespace latinime { -class DynamicPatriciaTrieReadingUtils { +class DynamicPtReadingUtils { public: typedef uint8_t NodeFlags; @@ -71,7 +71,7 @@ class DynamicPatriciaTrieReadingUtils { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieReadingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtReadingUtils); static const NodeFlags MASK_MOVED; static const NodeFlags FLAG_IS_NOT_MOVED; @@ -80,4 +80,4 @@ class DynamicPatriciaTrieReadingUtils { static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL; }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp similarity index 88% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp index f12a8995e..2457b49c8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp @@ -14,21 +14,21 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { -const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; +const int DynamicPtUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; -bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( - DynamicPatriciaTrieReadingHelper *const readingHelper, +bool DynamicPtUpdatingHelper::addUnigramWord( + DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram) { @@ -86,7 +86,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( isNotAWord, isBlacklisted, probability, timestamp, &pos); } -bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, +bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, const int probability, const int timestamp, bool *const outAddedNewBigram) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); @@ -97,7 +97,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const } // Remove a bigram relation from word0Pos to word1Pos. -bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { +bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); const PtNodeParams targetPtNodeParams( @@ -105,7 +105,7 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams); } -bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, +bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos, const int *const targetCodePoints, const int targetCodePointCount, const int shortcutProbability) { const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos)); @@ -113,12 +113,12 @@ bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, shortcutProbability); } -bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, +bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, int *const forwardLinkFieldPos) { const int newPtNodeArrayPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, newPtNodeArrayPos, forwardLinkFieldPos)) { return false; } @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( +bool DynamicPtUpdatingHelper::setPtNodeProbability( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, bool *const outAddedNewUnigram) { @@ -154,7 +154,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( return true; } -bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( +bool DynamicPtUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( const PtNodeParams *const parentPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, const int *const codePoints, const int codePointCount) { @@ -166,12 +166,12 @@ bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode codePointCount, isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( +bool DynamicPtUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp) { int writingPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } @@ -182,7 +182,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( &writingPos)) { return false; } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -190,7 +190,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( } // Returns whether the dictionary updating was succeeded or not. -bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( +bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode, const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) { @@ -227,7 +227,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const int actualChildrenPos = writingPos; // Create new children PtNode array. const size_t newPtNodeCount = addsExtraChild ? 2 : 1; - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, newPtNodeCount, &writingPos)) { return false; } @@ -252,7 +252,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return false; } } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -268,7 +268,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( +const PtNodeParams DynamicPtUpdatingHelper::getUpdatedPtNodeParams( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { @@ -280,7 +280,7 @@ const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( probability); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getPtNodeParamsForNewPtNode( +const PtNodeParams DynamicPtUpdatingHelper::getPtNodeParamsForNewPtNode( const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h similarity index 89% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h index f02635fe2..71f473096 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_UPDATING_HELPER_H +#define LATINIME_DYNAMIC_PT_UPDATING_HELPER_H #include @@ -26,21 +26,20 @@ namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieReadingHelper; +class DynamicPtReadingHelper; class PtNodeReader; class PtNodeWriter; -// TODO: Move to pt_common. -class DynamicPatriciaTrieUpdatingHelper { +class DynamicPtUpdatingHelper { public: - DynamicPatriciaTrieUpdatingHelper(BufferWithExtendableBuffer *const buffer, + DynamicPtUpdatingHelper(BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter) : mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {} - ~DynamicPatriciaTrieUpdatingHelper() {} + ~DynamicPtUpdatingHelper() {} // Add a word to the dictionary. If the word already exists, update the probability. - bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, + bool addUnigramWord(DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram); @@ -57,7 +56,7 @@ class DynamicPatriciaTrieUpdatingHelper { const int targetCodePointCount, const int shortcutProbability); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieUpdatingHelper); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtUpdatingHelper); static const int CHILDREN_POSITION_FIELD_SIZE; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp similarity index 68% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp index 45eeefd4c..ebbdc2ea2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include #include @@ -24,18 +24,18 @@ namespace latinime { -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; -const int DynamicPatriciaTrieWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; -const int DynamicPatriciaTrieWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; -const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; +const int DynamicPtWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; +const int DynamicPtWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; +const int DynamicPtWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; +const int DynamicPtWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; +const int DynamicPtWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; +const int DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE = 1; -/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( +/* static */ bool DynamicPtWritingUtils::writeEmptyDictionary( BufferWithExtendableBuffer *const buffer, const int rootPos) { int writingPos = rootPos; if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) { @@ -45,13 +45,13 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; &writingPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos) { return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const size_t arraySize, int *const arraySizeFieldPos) { // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to @@ -73,20 +73,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; } } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeFlagsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); } // Note that parentOffset is offset from node's head position. -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos, int *const parentPosFieldPos) { return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeCodePointsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int *const codePoints, const int codePointCount, int *const codePointFieldPos) { if (codePointCount <= 0) { @@ -100,21 +100,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; hasMultipleCodePoints, codePointFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int childrenPosition, int *const childrenPositionFieldPos) { return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos), childrenPositionFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset( - BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos, - int *const offsetFieldPos) { +/* static */ bool DynamicPtWritingUtils::writeDictOffset(BufferWithExtendableBuffer *const buffer, + const int targetPos, const int basePos, int *const offsetFieldPos) { int offset = targetPos - basePos; if (targetPos == NOT_A_DICT_POS) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; + offset = DynamicPtReadingUtils::DICT_OFFSET_INVALID; } else if (offset == 0) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; + offset = DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET; } if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) { AKLOGI("offset cannot be written because the offset is too large or too small: %d", diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h similarity index 84% rename from native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h index 43e8ea649..362fbd1cc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h @@ -14,19 +14,19 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_WRITING_UTILS_H +#define LATINIME_DYNAMIC_PT_WRITING_UTILS_H #include #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieWritingUtils { +class DynamicPtWritingUtils { public: static const int NODE_FLAG_FIELD_SIZE; @@ -40,14 +40,14 @@ class DynamicPatriciaTrieWritingUtils { const size_t arraySize, int *const arraySizeFieldPos); static bool writeFlags(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, const int nodeFlagsFieldPos) { int writingPos = nodeFlagsFieldPos; return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos); } static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, @@ -60,7 +60,7 @@ class DynamicPatriciaTrieWritingUtils { const int childrenPosition, int *const childrenPositionFieldPos); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtWritingUtils); static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD; static const size_t MAX_PTNODE_ARRAY_SIZE; @@ -76,4 +76,4 @@ class DynamicPatriciaTrieWritingUtils { const int basePos, int *const offsetFieldPos); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_WRITING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h index 2a43e39bf..84731eb17 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h @@ -20,8 +20,8 @@ #include #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" namespace latinime { @@ -111,11 +111,11 @@ class PtNodeParams { // Flags AK_FORCE_INLINE bool isDeleted() const { - return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags); + return DynamicPtReadingUtils::isDeleted(mFlags); } AK_FORCE_INLINE bool willBecomeNonTerminal() const { - return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags); + return DynamicPtReadingUtils::willBecomeNonTerminal(mFlags); } AK_FORCE_INLINE bool hasChildren() const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h index 8420ee95a..b28f58336 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h @@ -23,6 +23,7 @@ namespace latinime { +// TODO: Move to pt_common class PatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index c16e0bdcb..d6d22c5c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -21,6 +21,7 @@ namespace latinime { +// TODO: Create PtConstants under the pt_common and move some constant values there. // Note that there are corresponding definitions in FormatSpec.java. class Ver4DictConstants { public: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 8ea029076..17fc9483b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -16,8 +16,8 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" @@ -45,10 +45,10 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce const PatriciaTrieReadingUtils::NodeFlags flags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPosOffset = - DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( + DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( dictBuf, &pos); const int parentPos = - DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos); + DynamicPtReadingUtils::getParentPtNodePos(parentPosOffset, headPos); int codePoints[MAX_WORD_LENGTH]; const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos); @@ -74,7 +74,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce if (usesAdditionalBuffer) { childrenPosFieldPos += mBuffer->getOriginalBufferSize(); } - int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + int childrenPos = DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, &pos); if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) { childrenPos += mBuffer->getOriginalBufferSize(); @@ -85,7 +85,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce // Sibling position is the tail position of original PtNode. int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos; // Read destination node if the read node is a moved node. - if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { + if (DynamicPtReadingUtils::isMoved(flags)) { // The destination position is stored at the same place as the parent position. return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos); } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 10f90523a..32576cf0a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -19,11 +19,11 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" @@ -44,11 +44,11 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, true /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } @@ -74,16 +74,16 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, false /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } // Update moved position, which is stored in the parent offset field. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -93,8 +93,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( while (!mReadingHelper.isEnd()) { const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams()); int parentOffsetFieldPos = childPtNodeParams.getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(), &parentOffsetFieldPos)) { // Parent offset cannot be written because of a bug or a broken dictionary; thus, @@ -119,7 +119,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, false /* isDeleted */, true /* willBecomeNonTerminal */); if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) { @@ -129,7 +129,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( } // Update flags. int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); - return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos); } @@ -186,7 +186,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition( const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) { int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos(); - return DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, newChildrenPosition, &childrenPosFieldPos); } @@ -264,9 +264,9 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields( } } int writingPos = toBeUpdatedPtNodeParams->getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; // Write updated parent offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -328,17 +328,17 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( const int nodePos = *ptNodeWritingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, 0 /* nodeFlags */, ptNodeWritingPos)) { return false; } // Calculate a parent offset and write the offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) { return false; } // Write code points - if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) { return false; } @@ -369,7 +369,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( } } // Write children position - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, ptNodeParams->getChildrenPos(), ptNodeWritingPos)) { return false; } @@ -401,7 +401,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal, hasShortcutTargets, hasBigrams, hasMultipleChars, CHILDREN_POSITION_FIELD_SIZE); - if (!DynamicPatriciaTrieWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { + if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 1df853a30..69576d8e5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -20,9 +20,9 @@ #include #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" @@ -115,7 +115,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { BufferWithExtendableBuffer *const mTrieBuffer; Ver4DictBuffers *const mBuffers; const Ver4PatriciaTrieNodeReader *const mPtNodeReader; - DynamicPatriciaTrieReadingHelper mReadingHelper; + DynamicPtReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 7b5461d74..96bb8128e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -21,7 +21,7 @@ #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dictionary/unigram_property.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" @@ -48,7 +48,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d if (!dicNode->hasChildren()) { return; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); while (!readingHelper.isEnd()) { const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams(); @@ -75,7 +75,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodePos(ptNodePos); return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount( maxCodePointCount, outCodePoints, outUnigramProbability); @@ -83,7 +83,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); } @@ -154,7 +154,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len mDictBuffer->getTailPosition()); return false; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); bool addedNewUnigram = false; if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 2d46c0ae2..8187b7a39 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -22,7 +22,7 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" @@ -131,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { Ver4ShortcutListPolicy mShortcutPolicy; Ver4PatriciaTrieNodeReader mNodeReader; Ver4PatriciaTrieNodeWriter mNodeWriter; - DynamicPatriciaTrieUpdatingHelper mUpdatingHelper; + DynamicPtUpdatingHelper mUpdatingHelper; Ver4PatriciaTrieWritingHelper mWritingHelper; int mUnigramCount; int mBigramCount; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 2ee9483f7..25f87efae 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -88,9 +88,9 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); + DynamicPtReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners + DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( &ptNodeWriter); @@ -111,7 +111,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, } readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability + DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability traversePolicyToUpdateBigramProbability(&ptNodeWriter); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateBigramProbability)) { @@ -132,7 +132,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer + DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( @@ -170,10 +170,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, mBuffers->getShortcutDictContent())) { return false; } - DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), + DynamicPtReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), &newPtNodeReader); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields + DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap); if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( &traversePolicyToUpdateAllPositionFields)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h index 73e63ef45..198c10878 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h @@ -18,7 +18,7 @@ #define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" namespace latinime { @@ -43,7 +43,7 @@ class Ver4PatriciaTrieWritingHelper { DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds( Ver4PatriciaTrieNodeWriter *const ptNodeWriter, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index 9223356c1..ec67c188f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -19,7 +19,7 @@ #include #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" @@ -51,7 +51,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); - if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( + if (!DynamicPtWritingUtils::writeEmptyDictionary( dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) { AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp index e3d783835..cd3c403fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp @@ -46,8 +46,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; // same so we use them for both here. if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) { return VERSION_2; - } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_3) { - return VERSION_3; } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { return VERSION_4; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 34727b8d8..eb2227d60 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -31,7 +31,6 @@ class FormatUtils { enum FORMAT_VERSION { // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, - VERSION_3 = 3, VERSION_4 = 400, UNKNOWN_VERSION = -1 };