nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 4e9d1154: Merge "Remove or rename native codes for ver3 dictionary format." 

* commit '4e9d115401ce4deec7308849d63e0e64a3ed24cd':
  Remove or rename native codes for ver3 dictionary format.
main
Keisuke Kuroyanagi 2013-12-11 22:33:20 -08:00 committed by Android Git Automerger
parent 96401923d1 4e9d115401
commit 69b849f05e
27 changed files with 173 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
native/jni/Android.mk
View File

@ -76,15 +76,15 @@ LATIN_IME_CORE_SRC_FILES := \
$(addprefix suggest/policyimpl/dictionary/bigram/, \ $(addprefix suggest/policyimpl/dictionary/bigram/, \
bigram_list_read_write_utils.cpp \ bigram_list_read_write_utils.cpp \
ver4_bigram_list_policy.cpp) \ ver4_bigram_list_policy.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
dynamic_pt_gc_event_listeners.cpp \
dynamic_pt_reading_helper.cpp \
dynamic_pt_reading_utils.cpp \
dynamic_pt_updating_helper.cpp \
dynamic_pt_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v2/, \ $(addprefix suggest/policyimpl/dictionary/structure/v2/, \
patricia_trie_policy.cpp \ patricia_trie_policy.cpp \
patricia_trie_reading_utils.cpp) \ patricia_trie_reading_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v3/, \
dynamic_patricia_trie_gc_event_listeners.cpp \
dynamic_patricia_trie_reading_helper.cpp \
dynamic_patricia_trie_reading_utils.cpp \
dynamic_patricia_trie_updating_helper.cpp \
dynamic_patricia_trie_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \ $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
ver4_dict_buffers.cpp \ ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \ ver4_dict_constants.cpp \

6
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp
View File

@ -16,7 +16,6 @@
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@ -78,11 +77,6 @@ const BigramListReadWriteUtils::BigramFlags
offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
break; break;
} }
if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) {
return NOT_A_DICT_POS;
} else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) {
return origin;
}
if (isOffsetNegative(flags)) { if (isOffsetNegative(flags)) {
return origin - offset; return origin - offset;
} else { } else {

2
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -87,8 +87,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
switch (mDictFormatVersion) { switch (mDictFormatVersion) {
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
return FormatUtils::VERSION_2; return FormatUtils::VERSION_2;
case FormatUtils::VERSION_3:
return FormatUtils::VERSION_3;
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
return FormatUtils::VERSION_4; return FormatUtils::VERSION_4;
default: default:

3
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -89,9 +89,6 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported. // Version 2 dictionary writing is not supported.
return false; return false;
case FormatUtils::VERSION_3:
return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_3 /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */, return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos); HEADER_DICTIONARY_VERSION_SIZE, writingPos);

22
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp
View File

@ -14,16 +14,16 @@
* limitations under the License. * limitations under the License.
*/ */
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h" #include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
namespace latinime { namespace latinime {
bool DynamicPatriciaTrieGcEventListeners bool DynamicPtGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// PtNode is useless when the PtNode is not a terminal and doesn't have any not useless // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
@ -63,7 +63,7 @@ bool DynamicPatriciaTrieGcEventListeners
return true; return true;
} }
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability bool DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) { if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) {
int bigramEntryCount = 0; int bigramEntryCount = 0;
@ -77,7 +77,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit
} }
// Writes dummy PtNode array size when the head of PtNode array is read. // Writes dummy PtNode array size when the head of PtNode array is read.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onDescend(const int ptNodeArrayPos) { ::onDescend(const int ptNodeArrayPos) {
mValidPtNodeCount = 0; mValidPtNodeCount = 0;
int writingPos = mBufferToWrite->getTailPosition(); int writingPos = mBufferToWrite->getTailPosition();
@ -86,21 +86,21 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
// Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.
// This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.
mPtNodeArraySizeFieldPos = writingPos; mPtNodeArraySizeFieldPos = writingPos;
return DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( return DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
mBufferToWrite, 0 /* arraySize */, &writingPos); mBufferToWrite, 0 /* arraySize */, &writingPos);
} }
// Write PtNode array terminal and actual PtNode array size. // Write PtNode array terminal and actual PtNode array size.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onReadingPtNodeArrayTail() { ::onReadingPtNodeArrayTail() {
int writingPos = mBufferToWrite->getTailPosition(); int writingPos = mBufferToWrite->getTailPosition();
// Write PtNode array terminal. // Write PtNode array terminal.
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(
mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false; return false;
} }
// Write actual PtNode array size. // Write actual PtNode array size.
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) { mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) {
return false; return false;
} }
@ -108,7 +108,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
} }
// Write valid PtNode to buffer and memorize mapping from the old position to the new position. // Write valid PtNode to buffer and memorize mapping from the old position to the new position.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
if (ptNodeParams->isDeleted()) { if (ptNodeParams->isDeleted()) {
// Current PtNode is not written in new buffer because it has been deleted. // Current PtNode is not written in new buffer because it has been deleted.
@ -126,7 +126,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos); return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos);
} }
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields bool DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// Updates parent position. // Updates parent position.
int bigramCount = 0; int bigramCount = 0;

21
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h
View File

@ -14,14 +14,14 @@
* limitations under the License. * limitations under the License.
*/ */
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H #ifndef LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H #define LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H
#include <vector> #include <vector>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/hash_map_compat.h" #include "utils/hash_map_compat.h"
@ -29,14 +29,13 @@ namespace latinime {
class PtNodeParams; class PtNodeParams;
// TODO: Move to pt_common. class DynamicPtGcEventListeners {
class DynamicPatriciaTrieGcEventListeners {
public: public:
// Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or
// not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC. // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC.
// TODO: Concatenate non-terminal PtNodes. // TODO: Concatenate non-terminal PtNodes.
class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPtReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
PtNodeWriter *const ptNodeWriter) PtNodeWriter *const ptNodeWriter)
@ -81,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners {
// Updates all bigram entries that are held by valid PtNodes. This removes useless bigram // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram
// entries. // entries.
class TraversePolicyToUpdateBigramProbability class TraversePolicyToUpdateBigramProbability
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPtReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter) TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter)
: mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {} : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {}
@ -106,7 +105,7 @@ class DynamicPatriciaTrieGcEventListeners {
}; };
class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPtReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(
PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite,
@ -134,7 +133,7 @@ class DynamicPatriciaTrieGcEventListeners {
}; };
class TraversePolicyToUpdateAllPositionFields class TraversePolicyToUpdateAllPositionFields
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPtReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter, TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter,
const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
@ -168,7 +167,7 @@ class DynamicPatriciaTrieGcEventListeners {
}; };
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtGcEventListeners);
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H */ #endif /* LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H */

26
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp
View File

@ -14,25 +14,25 @@
* limitations under the License. * limitations under the License.
*/ */
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "utils/char_utils.h" #include "utils/char_utils.h"
namespace latinime { namespace latinime {
// To avoid infinite loop caused by invalid or malicious forward links. // To avoid infinite loop caused by invalid or malicious forward links.
const int DynamicPatriciaTrieReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; const int DynamicPtReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const int DynamicPatriciaTrieReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; const int DynamicPtReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const size_t DynamicPatriciaTrieReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; const size_t DynamicPtReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH;
// Visits all PtNodes in post-order depth first manner. // Visits all PtNodes in post-order depth first manner.
// For example, visits c -> b -> y -> x -> a for the following dictionary: // For example, visits c -> b -> y -> x -> a for the following dictionary:
// a _ b _ c // a _ b _ c
// \ x _ y // \ x _ y
bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( bool DynamicPtReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner(
TraversingEventListener *const listener) { TraversingEventListener *const listener) {
bool alreadyVisitedChildren = false; bool alreadyVisitedChildren = false;
// Descend from the root to the root PtNode array. // Descend from the root to the root PtNode array.
@ -92,7 +92,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa
// For example, visits a -> b -> x -> c -> y for the following dictionary: // For example, visits a -> b -> x -> c -> y for the following dictionary:
// a _ b _ c // a _ b _ c
// \ x _ y // \ x _ y
bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( bool DynamicPtReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
TraversingEventListener *const listener) { TraversingEventListener *const listener) {
bool alreadyVisitedAllPtNodesInArray = false; bool alreadyVisitedAllPtNodesInArray = false;
bool alreadyVisitedChildren = false; bool alreadyVisitedChildren = false;
@ -169,7 +169,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor
return !isError(); return !isError();
} }
int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( int DynamicPtReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount(
const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) { const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) {
// This method traverses parent nodes from the terminal by following parent pointers; thus, // This method traverses parent nodes from the terminal by following parent pointers; thus,
// node code points are stored in the buffer in the reverse order. // node code points are stored in the buffer in the reverse order.
@ -211,7 +211,7 @@ int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePo
return totalCodePointCount; return totalCodePointCount;
} }
int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, int DynamicPtReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) { const int length, const bool forceLowerCaseSearch) {
int searchCodePoints[length]; int searchCodePoints[length];
for (int i = 0; i < length; ++i) { for (int i = 0; i < length; ++i) {
@ -257,7 +257,7 @@ int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int
// Read node array size and process empty node arrays. Nodes and arrays are counted up in this // Read node array size and process empty node arrays. Nodes and arrays are counted up in this
// method to avoid an infinite loop. // method to avoid an infinite loop.
void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { void DynamicPtReadingHelper::nextPtNodeArray() {
if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
// Reading invalid position because of a bug or a broken dictionary. // Reading invalid position because of a bug or a broken dictionary.
AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d", AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d",
@ -308,7 +308,7 @@ void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() {
} }
// Follow the forward link and read the next node array if exists. // Follow the forward link and read the next node array if exists.
void DynamicPatriciaTrieReadingHelper::followForwardLink() { void DynamicPtReadingHelper::followForwardLink() {
if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
// Reading invalid position because of bug or broken dictionary. // Reading invalid position because of bug or broken dictionary.
AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d", AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d",
@ -324,12 +324,12 @@ void DynamicPatriciaTrieReadingHelper::followForwardLink() {
mReadingState.mPos -= mBuffer->getOriginalBufferSize(); mReadingState.mPos -= mBuffer->getOriginalBufferSize();
} }
const int forwardLinkPosition = const int forwardLinkPosition =
DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); DynamicPtReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos);
if (usesAdditionalBuffer) { if (usesAdditionalBuffer) {
mReadingState.mPos += mBuffer->getOriginalBufferSize(); mReadingState.mPos += mBuffer->getOriginalBufferSize();
} }
mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos; mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos;
if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { if (DynamicPtReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) {
// Follow the forward link. // Follow the forward link.
mReadingState.mPos += forwardLinkPosition; mReadingState.mPos += forwardLinkPosition;
nextPtNodeArray(); nextPtNodeArray();

15
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h
View File

@ -14,8 +14,8 @@
* limitations under the License. * limitations under the License.
*/ */
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H #ifndef LATINIME_DYNAMIC_PT_READING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H #define LATINIME_DYNAMIC_PT_READING_HELPER_H
#include <cstddef> #include <cstddef>
#include <vector> #include <vector>
@ -34,8 +34,7 @@ class DictionaryShortcutsStructurePolicy;
* This class is used for traversing dynamic patricia trie. This class supports iterating nodes and * This class is used for traversing dynamic patricia trie. This class supports iterating nodes and
* dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop. * dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop.
*/ */
// TODO: Move to pt_common. class DynamicPtReadingHelper {
class DynamicPatriciaTrieReadingHelper {
public: public:
class TraversingEventListener { class TraversingEventListener {
public: public:
@ -60,12 +59,12 @@ class DynamicPatriciaTrieReadingHelper {
DISALLOW_COPY_AND_ASSIGN(TraversingEventListener); DISALLOW_COPY_AND_ASSIGN(TraversingEventListener);
}; };
DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer, DynamicPtReadingHelper(const BufferWithExtendableBuffer *const buffer,
const PtNodeReader *const ptNodeReader) const PtNodeReader *const ptNodeReader)
: mIsError(false), mReadingState(), mBuffer(buffer), : mIsError(false), mReadingState(), mBuffer(buffer),
mPtNodeReader(ptNodeReader), mReadingStateStack() {} mPtNodeReader(ptNodeReader), mReadingStateStack() {}
~DynamicPatriciaTrieReadingHelper() {} ~DynamicPtReadingHelper() {}
AK_FORCE_INLINE bool isError() const { AK_FORCE_INLINE bool isError() const {
return mIsError; return mIsError;
@ -205,7 +204,7 @@ class DynamicPatriciaTrieReadingHelper {
const bool forceLowerCaseSearch); const bool forceLowerCaseSearch);
private: private:
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieReadingHelper); DISALLOW_COPY_AND_ASSIGN(DynamicPtReadingHelper);
// This class encapsulates the reading state of a position in the dictionary. It points at a // This class encapsulates the reading state of a position in the dictionary. It points at a
// specific PtNode in the dictionary. // specific PtNode in the dictionary.
@ -267,4 +266,4 @@ class DynamicPatriciaTrieReadingHelper {
} }
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H */ #endif /* LATINIME_DYNAMIC_PT_READING_HELPER_H */

27
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp
View File

@ -14,39 +14,38 @@
* limitations under the License. * limitations under the License.
*/ */
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime { namespace latinime {
typedef DynamicPatriciaTrieReadingUtils DptReadingUtils; const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::MASK_MOVED = 0xC0;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_NOT_MOVED = 0xC0;
const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0; const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_MOVED = 0x40;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_DELETED = 0x80;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00;
// TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0.
// Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum
// value of offsets, which is 0x7FFFFF is used to represent 0 offset. // value of offsets, which is 0x7FFFFF is used to represent 0 offset.
const int DptReadingUtils::DICT_OFFSET_INVALID = 0; const int DynamicPtReadingUtils::DICT_OFFSET_INVALID = 0;
const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; const int DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, /* static */ int DynamicPtReadingUtils::getForwardLinkPosition(const uint8_t *const buffer,
const int pos) { const int pos) {
int linkAddressPos = pos; int linkAddressPos = pos;
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos);
} }
/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( /* static */ int DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
const uint8_t *const buffer, int *const pos) { const uint8_t *const buffer, int *const pos) {
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
} }
/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) { /* static */ int DynamicPtReadingUtils::getParentPtNodePos(const int parentOffset,
const int ptNodePos) {
if (parentOffset == DICT_OFFSET_INVALID) { if (parentOffset == DICT_OFFSET_INVALID) {
return NOT_A_DICT_POS; return NOT_A_DICT_POS;
} else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) { } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) {
@ -56,7 +55,7 @@ const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
} }
} }
/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( /* static */ int DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition(
const uint8_t *const buffer, int *const pos) { const uint8_t *const buffer, int *const pos) {
const int base = *pos; const int base = *pos;
const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);

10
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h
View File

@ -14,8 +14,8 @@
* limitations under the License. * limitations under the License.
*/ */
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H #ifndef LATINIME_DYNAMIC_PT_READING_UTILS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H #define LATINIME_DYNAMIC_PT_READING_UTILS_H
#include <stdint.h> #include <stdint.h>
@ -23,7 +23,7 @@
namespace latinime { namespace latinime {
class DynamicPatriciaTrieReadingUtils { class DynamicPtReadingUtils {
public: public:
typedef uint8_t NodeFlags; typedef uint8_t NodeFlags;
@ -71,7 +71,7 @@ class DynamicPatriciaTrieReadingUtils {
} }
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieReadingUtils); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtReadingUtils);
static const NodeFlags MASK_MOVED; static const NodeFlags MASK_MOVED;
static const NodeFlags FLAG_IS_NOT_MOVED; static const NodeFlags FLAG_IS_NOT_MOVED;
@ -80,4 +80,4 @@ class DynamicPatriciaTrieReadingUtils {
static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL; static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL;
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */ #endif /* LATINIME_DYNAMIC_PT_READING_UTILS_H */

42
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp
View File

@ -14,21 +14,21 @@
* limitations under the License. * limitations under the License.
*/ */
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime { namespace latinime {
const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; const int DynamicPtUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( bool DynamicPtUpdatingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper, DynamicPtReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability, const int *const wordCodePoints, const int codePointCount, const int probability,
const bool isNotAWord, const bool isBlacklisted, const int timestamp, const bool isNotAWord, const bool isBlacklisted, const int timestamp,
bool *const outAddedNewUnigram) { bool *const outAddedNewUnigram) {
@ -86,7 +86,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
isNotAWord, isBlacklisted, probability, timestamp, &pos); isNotAWord, isBlacklisted, probability, timestamp, &pos);
} }
bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
const int probability, const int timestamp, bool *const outAddedNewBigram) { const int probability, const int timestamp, bool *const outAddedNewBigram) {
const PtNodeParams sourcePtNodeParams( const PtNodeParams sourcePtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
@ -97,7 +97,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const
} }
// Remove a bigram relation from word0Pos to word1Pos. // Remove a bigram relation from word0Pos to word1Pos.
bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
const PtNodeParams sourcePtNodeParams( const PtNodeParams sourcePtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
const PtNodeParams targetPtNodeParams( const PtNodeParams targetPtNodeParams(
@ -105,7 +105,7 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co
return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams); return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams);
} }
bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos,
const int *const targetCodePoints, const int targetCodePointCount, const int *const targetCodePoints, const int targetCodePointCount,
const int shortcutProbability) { const int shortcutProbability) {
const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos)); const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos));
@ -113,12 +113,12 @@ bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos,
shortcutProbability); shortcutProbability);
} }
bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos,
const int *const nodeCodePoints, const int nodeCodePointCount, const int *const nodeCodePoints, const int nodeCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probability, const bool isNotAWord, const bool isBlacklisted, const int probability,
const int timestamp, int *const forwardLinkFieldPos) { const int timestamp, int *const forwardLinkFieldPos) {
const int newPtNodeArrayPos = mBuffer->getTailPosition(); const int newPtNodeArrayPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
newPtNodeArrayPos, forwardLinkFieldPos)) { newPtNodeArrayPos, forwardLinkFieldPos)) {
return false; return false;
} }
@ -126,7 +126,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const
isNotAWord, isBlacklisted, probability, timestamp); isNotAWord, isBlacklisted, probability, timestamp);
} }
bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( bool DynamicPtUpdatingHelper::setPtNodeProbability(
const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const PtNodeParams *const originalPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const int probability, const int timestamp, const bool isBlacklisted, const int probability, const int timestamp,
bool *const outAddedNewUnigram) { bool *const outAddedNewUnigram) {
@ -154,7 +154,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
return true; return true;
} }
bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( bool DynamicPtUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode(
const PtNodeParams *const parentPtNodeParams, const bool isNotAWord, const PtNodeParams *const parentPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const int probability, const int timestamp, const bool isBlacklisted, const int probability, const int timestamp,
const int *const codePoints, const int codePointCount) { const int *const codePoints, const int codePointCount) {
@ -166,12 +166,12 @@ bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode
codePointCount, isNotAWord, isBlacklisted, probability, timestamp); codePointCount, isNotAWord, isBlacklisted, probability, timestamp);
} }
bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( bool DynamicPtUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probability, const bool isNotAWord, const bool isBlacklisted, const int probability,
const int timestamp) { const int timestamp) {
int writingPos = mBuffer->getTailPosition(); int writingPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
1 /* arraySize */, &writingPos)) { 1 /* arraySize */, &writingPos)) {
return false; return false;
} }
@ -182,7 +182,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
&writingPos)) { &writingPos)) {
return false; return false;
} }
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false; return false;
} }
@ -190,7 +190,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
} }
// Returns whether the dictionary updating was succeeded or not. // Returns whether the dictionary updating was succeeded or not.
bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode, const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode,
const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) { const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) {
@ -227,7 +227,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
const int actualChildrenPos = writingPos; const int actualChildrenPos = writingPos;
// Create new children PtNode array. // Create new children PtNode array.
const size_t newPtNodeCount = addsExtraChild ? 2 : 1; const size_t newPtNodeCount = addsExtraChild ? 2 : 1;
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
newPtNodeCount, &writingPos)) { newPtNodeCount, &writingPos)) {
return false; return false;
} }
@ -252,7 +252,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
return false; return false;
} }
} }
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false; return false;
} }
@ -268,7 +268,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos); return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos);
} }
const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( const PtNodeParams DynamicPtUpdatingHelper::getUpdatedPtNodeParams(
const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const PtNodeParams *const originalPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const bool isTerminal, const int parentPos, const bool isBlacklisted, const bool isTerminal, const int parentPos,
const int codePointCount, const int *const codePoints, const int probability) const { const int codePointCount, const int *const codePoints, const int probability) const {
@ -280,7 +280,7 @@ const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams(
probability); probability);
} }
const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getPtNodeParamsForNewPtNode( const PtNodeParams DynamicPtUpdatingHelper::getPtNodeParamsForNewPtNode(
const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const bool isNotAWord, const bool isBlacklisted, const bool isTerminal,
const int parentPos, const int codePointCount, const int *const codePoints, const int parentPos, const int codePointCount, const int *const codePoints,
const int probability) const { const int probability) const {

17
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h
View File

@ -14,8 +14,8 @@
* limitations under the License. * limitations under the License.
*/ */
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H #ifndef LATINIME_DYNAMIC_PT_UPDATING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H #define LATINIME_DYNAMIC_PT_UPDATING_HELPER_H
#include <stdint.h> #include <stdint.h>
@ -26,21 +26,20 @@
namespace latinime { namespace latinime {
class BufferWithExtendableBuffer; class BufferWithExtendableBuffer;
class DynamicPatriciaTrieReadingHelper; class DynamicPtReadingHelper;
class PtNodeReader; class PtNodeReader;
class PtNodeWriter; class PtNodeWriter;
// TODO: Move to pt_common. class DynamicPtUpdatingHelper {
class DynamicPatriciaTrieUpdatingHelper {
public: public:
DynamicPatriciaTrieUpdatingHelper(BufferWithExtendableBuffer *const buffer, DynamicPtUpdatingHelper(BufferWithExtendableBuffer *const buffer,
const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter) const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter)
: mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {} : mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {}
~DynamicPatriciaTrieUpdatingHelper() {} ~DynamicPtUpdatingHelper() {}
// Add a word to the dictionary. If the word already exists, update the probability. // Add a word to the dictionary. If the word already exists, update the probability.
bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, bool addUnigramWord(DynamicPtReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability, const int *const wordCodePoints, const int codePointCount, const int probability,
const bool isNotAWord, const bool isBlacklisted, const int timestamp, const bool isNotAWord, const bool isBlacklisted, const int timestamp,
bool *const outAddedNewUnigram); bool *const outAddedNewUnigram);
@ -57,7 +56,7 @@ class DynamicPatriciaTrieUpdatingHelper {
const int targetCodePointCount, const int shortcutProbability); const int targetCodePointCount, const int shortcutProbability);
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieUpdatingHelper); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtUpdatingHelper);
static const int CHILDREN_POSITION_FIELD_SIZE; static const int CHILDREN_POSITION_FIELD_SIZE;

47
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp
View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include <cstddef> #include <cstddef>
#include <cstdlib> #include <cstdlib>
@ -24,18 +24,18 @@
namespace latinime { namespace latinime {
const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F;
const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF;
const int DynamicPatriciaTrieWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; const int DynamicPtWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1;
const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2;
const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000;
const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; const int DynamicPtWritingUtils::DICT_OFFSET_FIELD_SIZE = 3;
const int DynamicPatriciaTrieWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; const int DynamicPtWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF;
const int DynamicPatriciaTrieWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; const int DynamicPtWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF;
const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; const int DynamicPtWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000;
const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; const int DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( /* static */ bool DynamicPtWritingUtils::writeEmptyDictionary(
BufferWithExtendableBuffer *const buffer, const int rootPos) { BufferWithExtendableBuffer *const buffer, const int rootPos) {
int writingPos = rootPos; int writingPos = rootPos;
if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) { if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) {
@ -45,13 +45,13 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
&writingPos); &writingPos);
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos) { int *const forwardLinkFieldPos) {
return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos); return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos);
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const size_t arraySize, BufferWithExtendableBuffer *const buffer, const size_t arraySize,
int *const arraySizeFieldPos) { int *const arraySizeFieldPos) {
// Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to
@ -73,20 +73,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
} }
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writeFlagsAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) {
return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos);
} }
// Note that parentOffset is offset from node's head position. // Note that parentOffset is offset from node's head position.
/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos, BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos,
int *const parentPosFieldPos) { int *const parentPosFieldPos) {
return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos); return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos);
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int *const codePoints, BufferWithExtendableBuffer *const buffer, const int *const codePoints,
const int codePointCount, int *const codePointFieldPos) { const int codePointCount, int *const codePointFieldPos) {
if (codePointCount <= 0) { if (codePointCount <= 0) {
@ -100,21 +100,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
hasMultipleCodePoints, codePointFieldPos); hasMultipleCodePoints, codePointFieldPos);
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( /* static */ bool DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int childrenPosition, BufferWithExtendableBuffer *const buffer, const int childrenPosition,
int *const childrenPositionFieldPos) { int *const childrenPositionFieldPos) {
return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos), return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos),
childrenPositionFieldPos); childrenPositionFieldPos);
} }
/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset( /* static */ bool DynamicPtWritingUtils::writeDictOffset(BufferWithExtendableBuffer *const buffer,
BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos, const int targetPos, const int basePos, int *const offsetFieldPos) {
int *const offsetFieldPos) {
int offset = targetPos - basePos; int offset = targetPos - basePos;
if (targetPos == NOT_A_DICT_POS) { if (targetPos == NOT_A_DICT_POS) {
offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; offset = DynamicPtReadingUtils::DICT_OFFSET_INVALID;
} else if (offset == 0) { } else if (offset == 0) {
offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; offset = DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET;
} }
if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) { if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) {
AKLOGI("offset cannot be written because the offset is too large or too small: %d", AKLOGI("offset cannot be written because the offset is too large or too small: %d",

16
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h
View File

@ -14,19 +14,19 @@
* limitations under the License. * limitations under the License.
*/ */
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H #ifndef LATINIME_DYNAMIC_PT_WRITING_UTILS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H #define LATINIME_DYNAMIC_PT_WRITING_UTILS_H
#include <cstddef> #include <cstddef>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
namespace latinime { namespace latinime {
class BufferWithExtendableBuffer; class BufferWithExtendableBuffer;
class DynamicPatriciaTrieWritingUtils { class DynamicPtWritingUtils {
public: public:
static const int NODE_FLAG_FIELD_SIZE; static const int NODE_FLAG_FIELD_SIZE;
@ -40,14 +40,14 @@ class DynamicPatriciaTrieWritingUtils {
const size_t arraySize, int *const arraySizeFieldPos); const size_t arraySize, int *const arraySizeFieldPos);
static bool writeFlags(BufferWithExtendableBuffer *const buffer, static bool writeFlags(BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, const DynamicPtReadingUtils::NodeFlags nodeFlags,
const int nodeFlagsFieldPos) { const int nodeFlagsFieldPos) {
int writingPos = nodeFlagsFieldPos; int writingPos = nodeFlagsFieldPos;
return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos); return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos);
} }
static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, const DynamicPtReadingUtils::NodeFlags nodeFlags,
int *const nodeFlagsFieldPos); int *const nodeFlagsFieldPos);
static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
@ -60,7 +60,7 @@ class DynamicPatriciaTrieWritingUtils {
const int childrenPosition, int *const childrenPositionFieldPos); const int childrenPosition, int *const childrenPositionFieldPos);
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingUtils); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtWritingUtils);
static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD; static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD;
static const size_t MAX_PTNODE_ARRAY_SIZE; static const size_t MAX_PTNODE_ARRAY_SIZE;
@ -76,4 +76,4 @@ class DynamicPatriciaTrieWritingUtils {
const int basePos, int *const offsetFieldPos); const int basePos, int *const offsetFieldPos);
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */ #endif /* LATINIME_DYNAMIC_PT_WRITING_UTILS_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h
View File

@ -20,8 +20,8 @@
#include <cstring> #include <cstring>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
namespace latinime { namespace latinime {
@ -111,11 +111,11 @@ class PtNodeParams {
// Flags // Flags
AK_FORCE_INLINE bool isDeleted() const { AK_FORCE_INLINE bool isDeleted() const {
return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags); return DynamicPtReadingUtils::isDeleted(mFlags);
} }
AK_FORCE_INLINE bool willBecomeNonTerminal() const { AK_FORCE_INLINE bool willBecomeNonTerminal() const {
return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags); return DynamicPtReadingUtils::willBecomeNonTerminal(mFlags);
} }
AK_FORCE_INLINE bool hasChildren() const { AK_FORCE_INLINE bool hasChildren() const {

1
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h
View File

@ -23,6 +23,7 @@
namespace latinime { namespace latinime {
// TODO: Move to pt_common
class PatriciaTrieReadingUtils { class PatriciaTrieReadingUtils {
public: public:
typedef uint8_t NodeFlags; typedef uint8_t NodeFlags;

1
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -21,6 +21,7 @@
namespace latinime { namespace latinime {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java. // Note that there are corresponding definitions in FormatSpec.java.
class Ver4DictConstants { class Ver4DictConstants {
public: public:

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
View File

@ -16,8 +16,8 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h"
@ -45,10 +45,10 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
const PatriciaTrieReadingUtils::NodeFlags flags = const PatriciaTrieReadingUtils::NodeFlags flags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const int parentPosOffset = const int parentPosOffset =
DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
dictBuf, &pos); dictBuf, &pos);
const int parentPos = const int parentPos =
DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos); DynamicPtReadingUtils::getParentPtNodePos(parentPosOffset, headPos);
int codePoints[MAX_WORD_LENGTH]; int codePoints[MAX_WORD_LENGTH];
const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos); dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos);
@ -74,7 +74,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
if (usesAdditionalBuffer) { if (usesAdditionalBuffer) {
childrenPosFieldPos += mBuffer->getOriginalBufferSize(); childrenPosFieldPos += mBuffer->getOriginalBufferSize();
} }
int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( int childrenPos = DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition(
dictBuf, &pos); dictBuf, &pos);
if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) { if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) {
childrenPos += mBuffer->getOriginalBufferSize(); childrenPos += mBuffer->getOriginalBufferSize();
@ -85,7 +85,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
// Sibling position is the tail position of original PtNode. // Sibling position is the tail position of original PtNode.
int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos; int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos;
// Read destination node if the read node is a moved node. // Read destination node if the read node is a moved node.
if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { if (DynamicPtReadingUtils::isMoved(flags)) {
// The destination position is stored at the same place as the parent position. // The destination position is stored at the same place as the parent position.
return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos); return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos);
} else { } else {

38
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
View File

@ -19,11 +19,11 @@
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
@ -44,11 +44,11 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted(
const PatriciaTrieReadingUtils::NodeFlags originalFlags = const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags = const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
true /* isDeleted */, false /* willBecomeNonTerminal */); true /* isDeleted */, false /* willBecomeNonTerminal */);
int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
// Update flags. // Update flags.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos)) { &writingPos)) {
return false; return false;
} }
@ -74,16 +74,16 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
const PatriciaTrieReadingUtils::NodeFlags originalFlags = const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags = const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */,
false /* isDeleted */, false /* willBecomeNonTerminal */); false /* isDeleted */, false /* willBecomeNonTerminal */);
int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
// Update flags. // Update flags.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos)) { &writingPos)) {
return false; return false;
} }
// Update moved position, which is stored in the parent offset field. // Update moved position, which is stored in the parent offset field.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false; return false;
} }
@ -93,8 +93,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
while (!mReadingHelper.isEnd()) { while (!mReadingHelper.isEnd()) {
const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams()); const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams());
int parentOffsetFieldPos = childPtNodeParams.getHeadPos() int parentOffsetFieldPos = childPtNodeParams.getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE;
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(), mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(),
&parentOffsetFieldPos)) { &parentOffsetFieldPos)) {
// Parent offset cannot be written because of a bug or a broken dictionary; thus, // Parent offset cannot be written because of a bug or a broken dictionary; thus,
@ -119,7 +119,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal(
const PatriciaTrieReadingUtils::NodeFlags originalFlags = const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags = const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
false /* isDeleted */, true /* willBecomeNonTerminal */); false /* isDeleted */, true /* willBecomeNonTerminal */);
if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition(
toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) { toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) {
@ -129,7 +129,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal(
} }
// Update flags. // Update flags.
int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos); &writingPos);
} }
@ -186,7 +186,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA
bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition( bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition(
const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) { const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) {
int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos(); int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos();
return DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
newChildrenPosition, &childrenPosFieldPos); newChildrenPosition, &childrenPosFieldPos);
} }
@ -264,9 +264,9 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
} }
} }
int writingPos = toBeUpdatedPtNodeParams->getHeadPos() int writingPos = toBeUpdatedPtNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write updated parent offset. // Write updated parent offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false; return false;
} }
@ -328,17 +328,17 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition(
const int nodePos = *ptNodeWritingPos; const int nodePos = *ptNodeWritingPos;
// Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the
// PtNode writing. // PtNode writing.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer,
0 /* nodeFlags */, ptNodeWritingPos)) { 0 /* nodeFlags */, ptNodeWritingPos)) {
return false; return false;
} }
// Calculate a parent offset and write the offset. // Calculate a parent offset and write the offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) { ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) {
return false; return false;
} }
// Write code points // Write code points
if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer,
ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) { ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) {
return false; return false;
} }
@ -369,7 +369,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition(
} }
} }
// Write children position // Write children position
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
ptNodeParams->getChildrenPos(), ptNodeWritingPos)) { ptNodeParams->getChildrenPos(), ptNodeWritingPos)) {
return false; return false;
} }
@ -401,7 +401,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos,
PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal, PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal,
hasShortcutTargets, hasBigrams, hasMultipleChars, hasShortcutTargets, hasBigrams, hasMultipleChars,
CHILDREN_POSITION_FIELD_SIZE); CHILDREN_POSITION_FIELD_SIZE);
if (!DynamicPatriciaTrieWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) {
AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos); AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos);
return false; return false;
} }

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
View File

@ -20,9 +20,9 @@
#include <stdint.h> #include <stdint.h>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
@ -115,7 +115,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
BufferWithExtendableBuffer *const mTrieBuffer; BufferWithExtendableBuffer *const mTrieBuffer;
Ver4DictBuffers *const mBuffers; Ver4DictBuffers *const mBuffers;
const Ver4PatriciaTrieNodeReader *const mPtNodeReader; const Ver4PatriciaTrieNodeReader *const mPtNodeReader;
DynamicPatriciaTrieReadingHelper mReadingHelper; DynamicPtReadingHelper mReadingHelper;
Ver4BigramListPolicy *const mBigramPolicy; Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy;
}; };

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View File

@ -21,7 +21,7 @@
#include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dicnode/dic_node_vector.h"
#include "suggest/core/dictionary/unigram_property.h" #include "suggest/core/dictionary/unigram_property.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h"
@ -48,7 +48,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
if (!dicNode->hasChildren()) { if (!dicNode->hasChildren()) {
return; return;
} }
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos());
while (!readingHelper.isEnd()) { while (!readingHelper.isEnd()) {
const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams(); const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams();
@ -75,7 +75,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, const int ptNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const { int *const outUnigramProbability) const {
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodePos(ptNodePos); readingHelper.initWithPtNodePos(ptNodePos);
return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount( return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount(
maxCodePointCount, outCodePoints, outUnigramProbability); maxCodePointCount, outCodePoints, outUnigramProbability);
@ -83,7 +83,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) const { const int length, const bool forceLowerCaseSearch) const {
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
} }
@ -154,7 +154,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
mDictBuffer->getTailPosition()); mDictBuffer->getTailPosition());
return false; return false;
} }
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition()); readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false; bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord, if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord,

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View File

@ -22,7 +22,7 @@
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h"
@ -131,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
Ver4ShortcutListPolicy mShortcutPolicy; Ver4ShortcutListPolicy mShortcutPolicy;
Ver4PatriciaTrieNodeReader mNodeReader; Ver4PatriciaTrieNodeReader mNodeReader;
Ver4PatriciaTrieNodeWriter mNodeWriter; Ver4PatriciaTrieNodeWriter mNodeWriter;
DynamicPatriciaTrieUpdatingHelper mUpdatingHelper; DynamicPtUpdatingHelper mUpdatingHelper;
Ver4PatriciaTrieWritingHelper mWritingHelper; Ver4PatriciaTrieWritingHelper mWritingHelper;
int mUnigramCount; int mUnigramCount;
int mBigramCount; int mBigramCount;

12
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
View File

@ -88,9 +88,9 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy); mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); DynamicPtReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners DynamicPtGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
&ptNodeWriter); &ptNodeWriter);
@ -111,7 +111,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
} }
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability
traversePolicyToUpdateBigramProbability(&ptNodeWriter); traversePolicyToUpdateBigramProbability(&ptNodeWriter);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateBigramProbability)) { &traversePolicyToUpdateBigramProbability)) {
@ -132,7 +132,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(),
buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy); buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers,
buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap);
if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
@ -170,10 +170,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
mBuffers->getShortcutDictContent())) { mBuffers->getShortcutDictContent())) {
return false; return false;
} }
DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), DynamicPtReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(),
&newPtNodeReader); &newPtNodeReader);
newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields
traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap); traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap);
if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
&traversePolicyToUpdateAllPositionFields)) { &traversePolicyToUpdateAllPositionFields)) {

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
View File

@ -18,7 +18,7 @@
#define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
namespace latinime { namespace latinime {
@ -43,7 +43,7 @@ class Ver4PatriciaTrieWritingHelper {
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener { : public DynamicPtReadingHelper::TraversingEventListener {
public: public:
TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds( TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds(
Ver4PatriciaTrieNodeWriter *const ptNodeWriter, Ver4PatriciaTrieNodeWriter *const ptNodeWriter,

4
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -19,7 +19,7 @@
#include <cstdio> #include <cstdio>
#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h"
@ -51,7 +51,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( if (!DynamicPtWritingUtils::writeEmptyDictionary(
dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) { dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) {
AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
return false; return false;

2
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View File

@ -46,8 +46,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
// same so we use them for both here. // same so we use them for both here.
if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) { if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
return VERSION_2; return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_3) {
return VERSION_3;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4; return VERSION_4;
} else { } else {

1
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View File

@ -31,7 +31,6 @@ class FormatUtils {
enum FORMAT_VERSION { enum FORMAT_VERSION {
// These MUST have the same values as the relevant constants in FormatSpec.java. // These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2, VERSION_2 = 2,
VERSION_3 = 3,
VERSION_4 = 400, VERSION_4 = 400,
UNKNOWN_VERSION = -1 UNKNOWN_VERSION = -1
}; };