nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Remove or rename native codes for ver3 dictionary format."

main
Keisuke Kuroyanagi 2013-12-12 06:31:59 +00:00 committed by Android (Google) Code Review
parent 9fa221a7bd 90cff563a0
commit 4e9d115401
27 changed files with 173 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
native/jni/Android.mk
View File

@ -76,15 +76,15 @@ LATIN_IME_CORE_SRC_FILES := \
$(addprefix suggest/policyimpl/dictionary/bigram/, \
bigram_list_read_write_utils.cpp \
ver4_bigram_list_policy.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \
dynamic_pt_gc_event_listeners.cpp \
dynamic_pt_reading_helper.cpp \
dynamic_pt_reading_utils.cpp \
dynamic_pt_updating_helper.cpp \
dynamic_pt_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v2/, \
patricia_trie_policy.cpp \
patricia_trie_reading_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v3/, \
dynamic_patricia_trie_gc_event_listeners.cpp \
dynamic_patricia_trie_reading_helper.cpp \
dynamic_patricia_trie_reading_utils.cpp \
dynamic_patricia_trie_updating_helper.cpp \
dynamic_patricia_trie_writing_utils.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \
ver4_dict_buffers.cpp \
ver4_dict_constants.cpp \

6
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp
View File

@ -16,7 +16,6 @@
#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@ -78,11 +77,6 @@ const BigramListReadWriteUtils::BigramFlags
offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
break;
}
if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) {
return NOT_A_DICT_POS;
} else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) {
return origin;
}
if (isOffsetNegative(flags)) {
return origin - offset;
} else {

2
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -87,8 +87,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
switch (mDictFormatVersion) {
case FormatUtils::VERSION_2:
return FormatUtils::VERSION_2;
case FormatUtils::VERSION_3:
return FormatUtils::VERSION_3;
case FormatUtils::VERSION_4:
return FormatUtils::VERSION_4;
default:

3
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -89,9 +89,6 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported.
return false;
case FormatUtils::VERSION_3:
return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_3 /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
case FormatUtils::VERSION_4:
return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos);

22
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp
View File

@ -14,16 +14,16 @@
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
namespace latinime {
bool DynamicPatriciaTrieGcEventListeners
bool DynamicPtGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
@ -63,7 +63,7 @@ bool DynamicPatriciaTrieGcEventListeners
return true;
}
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
bool DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) {
int bigramEntryCount = 0;
@ -77,7 +77,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit
}
// Writes dummy PtNode array size when the head of PtNode array is read.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onDescend(const int ptNodeArrayPos) {
mValidPtNodeCount = 0;
int writingPos = mBufferToWrite->getTailPosition();
@ -86,21 +86,21 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
// Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.
// This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.
mPtNodeArraySizeFieldPos = writingPos;
return DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(
return DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
mBufferToWrite, 0 /* arraySize */, &writingPos);
}
// Write PtNode array terminal and actual PtNode array size.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onReadingPtNodeArrayTail() {
int writingPos = mBufferToWrite->getTailPosition();
// Write PtNode array terminal.
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(
if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(
mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false;
}
// Write actual PtNode array size.
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(
if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) {
return false;
}
@ -108,7 +108,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
}
// Write valid PtNode to buffer and memorize mapping from the old position to the new position.
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
if (ptNodeParams->isDeleted()) {
// Current PtNode is not written in new buffer because it has been deleted.
@ -126,7 +126,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo
return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos);
}
bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
bool DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields
::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
// Updates parent position.
int bigramCount = 0;

21
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h
View File

@ -14,14 +14,14 @@
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H
#ifndef LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H
#define LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H
#include <vector>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "utils/hash_map_compat.h"
@ -29,14 +29,13 @@ namespace latinime {
class PtNodeParams;
// TODO: Move to pt_common.
class DynamicPatriciaTrieGcEventListeners {
class DynamicPtGcEventListeners {
public:
// Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or
// not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC.
// TODO: Concatenate non-terminal PtNodes.
class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
: public DynamicPtReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
PtNodeWriter *const ptNodeWriter)
@ -81,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners {
// Updates all bigram entries that are held by valid PtNodes. This removes useless bigram
// entries.
class TraversePolicyToUpdateBigramProbability
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
: public DynamicPtReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter)
: mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {}
@ -106,7 +105,7 @@ class DynamicPatriciaTrieGcEventListeners {
};
class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
: public DynamicPtReadingHelper::TraversingEventListener {
public:
TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(
PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite,
@ -134,7 +133,7 @@ class DynamicPatriciaTrieGcEventListeners {
};
class TraversePolicyToUpdateAllPositionFields
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
: public DynamicPtReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter,
const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap)
@ -168,7 +167,7 @@ class DynamicPatriciaTrieGcEventListeners {
};
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners);
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtGcEventListeners);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H */
#endif /* LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H */

26
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp
View File

@ -14,25 +14,25 @@
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "utils/char_utils.h"
namespace latinime {
// To avoid infinite loop caused by invalid or malicious forward links.
const int DynamicPatriciaTrieReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const int DynamicPatriciaTrieReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const size_t DynamicPatriciaTrieReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH;
const int DynamicPtReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const int DynamicPtReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000;
const size_t DynamicPtReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH;
// Visits all PtNodes in post-order depth first manner.
// For example, visits c -> b -> y -> x -> a for the following dictionary:
// a _ b _ c
// \ x _ y
bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner(
bool DynamicPtReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner(
TraversingEventListener *const listener) {
bool alreadyVisitedChildren = false;
// Descend from the root to the root PtNode array.
@ -92,7 +92,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa
// For example, visits a -> b -> x -> c -> y for the following dictionary:
// a _ b _ c
// \ x _ y
bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
bool DynamicPtReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
TraversingEventListener *const listener) {
bool alreadyVisitedAllPtNodesInArray = false;
bool alreadyVisitedChildren = false;
@ -169,7 +169,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor
return !isError();
}
int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount(
int DynamicPtReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount(
const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) {
// This method traverses parent nodes from the terminal by following parent pointers; thus,
// node code points are stored in the buffer in the reverse order.
@ -211,7 +211,7 @@ int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePo
return totalCodePointCount;
}
int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord,
int DynamicPtReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) {
int searchCodePoints[length];
for (int i = 0; i < length; ++i) {
@ -257,7 +257,7 @@ int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int
// Read node array size and process empty node arrays. Nodes and arrays are counted up in this
// method to avoid an infinite loop.
void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() {
void DynamicPtReadingHelper::nextPtNodeArray() {
if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
// Reading invalid position because of a bug or a broken dictionary.
AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d",
@ -308,7 +308,7 @@ void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() {
}
// Follow the forward link and read the next node array if exists.
void DynamicPatriciaTrieReadingHelper::followForwardLink() {
void DynamicPtReadingHelper::followForwardLink() {
if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
// Reading invalid position because of bug or broken dictionary.
AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d",
@ -324,12 +324,12 @@ void DynamicPatriciaTrieReadingHelper::followForwardLink() {
mReadingState.mPos -= mBuffer->getOriginalBufferSize();
}
const int forwardLinkPosition =
DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos);
DynamicPtReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos);
if (usesAdditionalBuffer) {
mReadingState.mPos += mBuffer->getOriginalBufferSize();
}
mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos;
if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) {
if (DynamicPtReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) {
// Follow the forward link.
mReadingState.mPos += forwardLinkPosition;
nextPtNodeArray();

15
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H
#ifndef LATINIME_DYNAMIC_PT_READING_HELPER_H
#define LATINIME_DYNAMIC_PT_READING_HELPER_H
#include <cstddef>
#include <vector>
@ -34,8 +34,7 @@ class DictionaryShortcutsStructurePolicy;
* This class is used for traversing dynamic patricia trie. This class supports iterating nodes and
* dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop.
*/
// TODO: Move to pt_common.
class DynamicPatriciaTrieReadingHelper {
class DynamicPtReadingHelper {
public:
class TraversingEventListener {
public:
@ -60,12 +59,12 @@ class DynamicPatriciaTrieReadingHelper {
DISALLOW_COPY_AND_ASSIGN(TraversingEventListener);
};
DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer,
DynamicPtReadingHelper(const BufferWithExtendableBuffer *const buffer,
const PtNodeReader *const ptNodeReader)
: mIsError(false), mReadingState(), mBuffer(buffer),
mPtNodeReader(ptNodeReader), mReadingStateStack() {}
~DynamicPatriciaTrieReadingHelper() {}
~DynamicPtReadingHelper() {}
AK_FORCE_INLINE bool isError() const {
return mIsError;
@ -205,7 +204,7 @@ class DynamicPatriciaTrieReadingHelper {
const bool forceLowerCaseSearch);
private:
DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieReadingHelper);
DISALLOW_COPY_AND_ASSIGN(DynamicPtReadingHelper);
// This class encapsulates the reading state of a position in the dictionary. It points at a
// specific PtNode in the dictionary.
@ -267,4 +266,4 @@ class DynamicPatriciaTrieReadingHelper {
}
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H */
#endif /* LATINIME_DYNAMIC_PT_READING_HELPER_H */

27
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp
View File

@ -14,39 +14,38 @@
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
namespace latinime {
typedef DynamicPatriciaTrieReadingUtils DptReadingUtils;
const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80;
const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::MASK_MOVED = 0xC0;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_NOT_MOVED = 0xC0;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_MOVED = 0x40;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_DELETED = 0x80;
const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00;
// TODO: Make DICT_OFFSET_ZERO_OFFSET = 0.
// Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum
// value of offsets, which is 0x7FFFFF is used to represent 0 offset.
const int DptReadingUtils::DICT_OFFSET_INVALID = 0;
const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
const int DynamicPtReadingUtils::DICT_OFFSET_INVALID = 0;
const int DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer,
/* static */ int DynamicPtReadingUtils::getForwardLinkPosition(const uint8_t *const buffer,
const int pos) {
int linkAddressPos = pos;
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos);
}
/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
/* static */ int DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
const uint8_t *const buffer, int *const pos) {
return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);
}
/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) {
/* static */ int DynamicPtReadingUtils::getParentPtNodePos(const int parentOffset,
const int ptNodePos) {
if (parentOffset == DICT_OFFSET_INVALID) {
return NOT_A_DICT_POS;
} else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) {
@ -56,7 +55,7 @@ const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF;
}
}
/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition(
/* static */ int DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const int base = *pos;
const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos);

10
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H
#ifndef LATINIME_DYNAMIC_PT_READING_UTILS_H
#define LATINIME_DYNAMIC_PT_READING_UTILS_H
#include <stdint.h>
@ -23,7 +23,7 @@
namespace latinime {
class DynamicPatriciaTrieReadingUtils {
class DynamicPtReadingUtils {
public:
typedef uint8_t NodeFlags;
@ -71,7 +71,7 @@ class DynamicPatriciaTrieReadingUtils {
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieReadingUtils);
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtReadingUtils);
static const NodeFlags MASK_MOVED;
static const NodeFlags FLAG_IS_NOT_MOVED;
@ -80,4 +80,4 @@ class DynamicPatriciaTrieReadingUtils {
static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL;
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */
#endif /* LATINIME_DYNAMIC_PT_READING_UTILS_H */

42
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp
View File

@ -14,21 +14,21 @@
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
const int DynamicPtUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
DynamicPatriciaTrieReadingHelper *const readingHelper,
bool DynamicPtUpdatingHelper::addUnigramWord(
DynamicPtReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability,
const bool isNotAWord, const bool isBlacklisted, const int timestamp,
bool *const outAddedNewUnigram) {
@ -86,7 +86,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord(
isNotAWord, isBlacklisted, probability, timestamp, &pos);
}
bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
const int probability, const int timestamp, bool *const outAddedNewBigram) {
const PtNodeParams sourcePtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
@ -97,7 +97,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const
}
// Remove a bigram relation from word0Pos to word1Pos.
bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
const PtNodeParams sourcePtNodeParams(
mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
const PtNodeParams targetPtNodeParams(
@ -105,7 +105,7 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co
return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams);
}
bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos,
bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos,
const int *const targetCodePoints, const int targetCodePointCount,
const int shortcutProbability) {
const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos));
@ -113,12 +113,12 @@ bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos,
shortcutProbability);
}
bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos,
bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos,
const int *const nodeCodePoints, const int nodeCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probability,
const int timestamp, int *const forwardLinkFieldPos) {
const int newPtNodeArrayPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
newPtNodeArrayPos, forwardLinkFieldPos)) {
return false;
}
@ -126,7 +126,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const
isNotAWord, isBlacklisted, probability, timestamp);
}
bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
bool DynamicPtUpdatingHelper::setPtNodeProbability(
const PtNodeParams *const originalPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const int probability, const int timestamp,
bool *const outAddedNewUnigram) {
@ -154,7 +154,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability(
return true;
}
bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode(
bool DynamicPtUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode(
const PtNodeParams *const parentPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const int probability, const int timestamp,
const int *const codePoints, const int codePointCount) {
@ -166,12 +166,12 @@ bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode
codePointCount, isNotAWord, isBlacklisted, probability, timestamp);
}
bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
bool DynamicPtUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probability,
const int timestamp) {
int writingPos = mBuffer->getTailPosition();
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
1 /* arraySize */, &writingPos)) {
return false;
}
@ -182,7 +182,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
&writingPos)) {
return false;
}
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false;
}
@ -190,7 +190,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode(
}
// Returns whether the dictionary updating was succeeded or not.
bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount,
const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode,
const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) {
@ -227,7 +227,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
const int actualChildrenPos = writingPos;
// Create new children PtNode array.
const size_t newPtNodeCount = addsExtraChild ? 2 : 1;
if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer,
newPtNodeCount, &writingPos)) {
return false;
}
@ -252,7 +252,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
return false;
}
}
if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer,
NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) {
return false;
}
@ -268,7 +268,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos);
}
const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams(
const PtNodeParams DynamicPtUpdatingHelper::getUpdatedPtNodeParams(
const PtNodeParams *const originalPtNodeParams, const bool isNotAWord,
const bool isBlacklisted, const bool isTerminal, const int parentPos,
const int codePointCount, const int *const codePoints, const int probability) const {
@ -280,7 +280,7 @@ const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams(
probability);
}
const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getPtNodeParamsForNewPtNode(
const PtNodeParams DynamicPtUpdatingHelper::getPtNodeParamsForNewPtNode(
const bool isNotAWord, const bool isBlacklisted, const bool isTerminal,
const int parentPos, const int codePointCount, const int *const codePoints,
const int probability) const {

17
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H
#ifndef LATINIME_DYNAMIC_PT_UPDATING_HELPER_H
#define LATINIME_DYNAMIC_PT_UPDATING_HELPER_H
#include <stdint.h>
@ -26,21 +26,20 @@
namespace latinime {
class BufferWithExtendableBuffer;
class DynamicPatriciaTrieReadingHelper;
class DynamicPtReadingHelper;
class PtNodeReader;
class PtNodeWriter;
// TODO: Move to pt_common.
class DynamicPatriciaTrieUpdatingHelper {
class DynamicPtUpdatingHelper {
public:
DynamicPatriciaTrieUpdatingHelper(BufferWithExtendableBuffer *const buffer,
DynamicPtUpdatingHelper(BufferWithExtendableBuffer *const buffer,
const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter)
: mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {}
~DynamicPatriciaTrieUpdatingHelper() {}
~DynamicPtUpdatingHelper() {}
// Add a word to the dictionary. If the word already exists, update the probability.
bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper,
bool addUnigramWord(DynamicPtReadingHelper *const readingHelper,
const int *const wordCodePoints, const int codePointCount, const int probability,
const bool isNotAWord, const bool isBlacklisted, const int timestamp,
bool *const outAddedNewUnigram);
@ -57,7 +56,7 @@ class DynamicPatriciaTrieUpdatingHelper {
const int targetCodePointCount, const int shortcutProbability);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieUpdatingHelper);
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtUpdatingHelper);
static const int CHILDREN_POSITION_FIELD_SIZE;

47
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include <cstddef>
#include <cstdlib>
@ -24,18 +24,18 @@
namespace latinime {
const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F;
const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF;
const int DynamicPatriciaTrieWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1;
const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2;
const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000;
const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_FIELD_SIZE = 3;
const int DynamicPatriciaTrieWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF;
const int DynamicPatriciaTrieWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF;
const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000;
const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F;
const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF;
const int DynamicPtWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1;
const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2;
const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000;
const int DynamicPtWritingUtils::DICT_OFFSET_FIELD_SIZE = 3;
const int DynamicPtWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF;
const int DynamicPtWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF;
const int DynamicPtWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000;
const int DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
/* static */ bool DynamicPtWritingUtils::writeEmptyDictionary(
BufferWithExtendableBuffer *const buffer, const int rootPos) {
int writingPos = rootPos;
if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) {
@ -45,13 +45,13 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
&writingPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
int *const forwardLinkFieldPos) {
return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const size_t arraySize,
int *const arraySizeFieldPos) {
// Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to
@ -73,20 +73,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
}
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writeFlagsAndAdvancePosition(
BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) {
const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) {
return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos);
}
// Note that parentOffset is offset from node's head position.
/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos,
int *const parentPosFieldPos) {
return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int *const codePoints,
const int codePointCount, int *const codePointFieldPos) {
if (codePointCount <= 0) {
@ -100,21 +100,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;
hasMultipleCodePoints, codePointFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(
/* static */ bool DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(
BufferWithExtendableBuffer *const buffer, const int childrenPosition,
int *const childrenPositionFieldPos) {
return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos),
childrenPositionFieldPos);
}
/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset(
BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos,
int *const offsetFieldPos) {
/* static */ bool DynamicPtWritingUtils::writeDictOffset(BufferWithExtendableBuffer *const buffer,
const int targetPos, const int basePos, int *const offsetFieldPos) {
int offset = targetPos - basePos;
if (targetPos == NOT_A_DICT_POS) {
offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID;
offset = DynamicPtReadingUtils::DICT_OFFSET_INVALID;
} else if (offset == 0) {
offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET;
offset = DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET;
}
if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) {
AKLOGI("offset cannot be written because the offset is too large or too small: %d",

16
native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h → native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h
View File

@ -14,19 +14,19 @@
* limitations under the License.
*/
#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H
#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H
#ifndef LATINIME_DYNAMIC_PT_WRITING_UTILS_H
#define LATINIME_DYNAMIC_PT_WRITING_UTILS_H
#include <cstddef>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
namespace latinime {
class BufferWithExtendableBuffer;
class DynamicPatriciaTrieWritingUtils {
class DynamicPtWritingUtils {
public:
static const int NODE_FLAG_FIELD_SIZE;
@ -40,14 +40,14 @@ class DynamicPatriciaTrieWritingUtils {
const size_t arraySize, int *const arraySizeFieldPos);
static bool writeFlags(BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags,
const DynamicPtReadingUtils::NodeFlags nodeFlags,
const int nodeFlagsFieldPos) {
int writingPos = nodeFlagsFieldPos;
return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos);
}
static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags,
const DynamicPtReadingUtils::NodeFlags nodeFlags,
int *const nodeFlagsFieldPos);
static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer,
@ -60,7 +60,7 @@ class DynamicPatriciaTrieWritingUtils {
const int childrenPosition, int *const childrenPositionFieldPos);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingUtils);
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtWritingUtils);
static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD;
static const size_t MAX_PTNODE_ARRAY_SIZE;
@ -76,4 +76,4 @@ class DynamicPatriciaTrieWritingUtils {
const int basePos, int *const offsetFieldPos);
};
} // namespace latinime
#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */
#endif /* LATINIME_DYNAMIC_PT_WRITING_UTILS_H */

6
native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h
View File

@ -20,8 +20,8 @@
#include <cstring>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
namespace latinime {
@ -111,11 +111,11 @@ class PtNodeParams {
// Flags
AK_FORCE_INLINE bool isDeleted() const {
return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags);
return DynamicPtReadingUtils::isDeleted(mFlags);
}
AK_FORCE_INLINE bool willBecomeNonTerminal() const {
return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags);
return DynamicPtReadingUtils::willBecomeNonTerminal(mFlags);
}
AK_FORCE_INLINE bool hasChildren() const {

1
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h
View File

@ -23,6 +23,7 @@
namespace latinime {
// TODO: Move to pt_common
class PatriciaTrieReadingUtils {
public:
typedef uint8_t NodeFlags;

1
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -21,6 +21,7 @@
namespace latinime {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java.
class Ver4DictConstants {
public:

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
View File

@ -16,8 +16,8 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h"
@ -45,10 +45,10 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
const PatriciaTrieReadingUtils::NodeFlags flags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const int parentPosOffset =
DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(
dictBuf, &pos);
const int parentPos =
DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos);
DynamicPtReadingUtils::getParentPtNodePos(parentPosOffset, headPos);
int codePoints[MAX_WORD_LENGTH];
const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition(
dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos);
@ -74,7 +74,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
if (usesAdditionalBuffer) {
childrenPosFieldPos += mBuffer->getOriginalBufferSize();
}
int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
int childrenPos = DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition(
dictBuf, &pos);
if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) {
childrenPos += mBuffer->getOriginalBufferSize();
@ -85,7 +85,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
// Sibling position is the tail position of original PtNode.
int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos;
// Read destination node if the read node is a moved node.
if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) {
if (DynamicPtReadingUtils::isMoved(flags)) {
// The destination position is stored at the same place as the parent position.
return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos);
} else {

38
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
View File

@ -19,11 +19,11 @@
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
@ -44,11 +44,11 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted(
const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
true /* isDeleted */, false /* willBecomeNonTerminal */);
int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
// Update flags.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos)) {
return false;
}
@ -74,16 +74,16 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */,
DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */,
false /* isDeleted */, false /* willBecomeNonTerminal */);
int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
// Update flags.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos)) {
return false;
}
// Update moved position, which is stored in the parent offset field.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false;
}
@ -93,8 +93,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved(
while (!mReadingHelper.isEnd()) {
const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams());
int parentOffsetFieldPos = childPtNodeParams.getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(
+ DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE;
if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(
mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(),
&parentOffsetFieldPos)) {
// Parent offset cannot be written because of a bug or a broken dictionary; thus,
@ -119,7 +119,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal(
const PatriciaTrieReadingUtils::NodeFlags originalFlags =
PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
const PatriciaTrieReadingUtils::NodeFlags updatedFlags =
DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */,
false /* isDeleted */, true /* willBecomeNonTerminal */);
if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition(
toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) {
@ -129,7 +129,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal(
}
// Update flags.
int writingPos = toBeUpdatedPtNodeParams->getHeadPos();
return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags,
&writingPos);
}
@ -186,7 +186,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA
bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition(
const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) {
int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos();
return DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
newChildrenPosition, &childrenPosFieldPos);
}
@ -264,9 +264,9 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
}
}
int writingPos = toBeUpdatedPtNodeParams->getHeadPos()
+ DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;
+ DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE;
// Write updated parent offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) {
return false;
}
@ -328,17 +328,17 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition(
const int nodePos = *ptNodeWritingPos;
// Write dummy flags. The Node flags are updated with appropriate flags at the last step of the
// PtNode writing.
if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer,
if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer,
0 /* nodeFlags */, ptNodeWritingPos)) {
return false;
}
// Calculate a parent offset and write the offset.
if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer,
ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) {
return false;
}
// Write code points
if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer,
if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer,
ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) {
return false;
}
@ -369,7 +369,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition(
}
}
// Write children position
if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer,
ptNodeParams->getChildrenPos(), ptNodeWritingPos)) {
return false;
}
@ -401,7 +401,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos,
PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal,
hasShortcutTargets, hasBigrams, hasMultipleChars,
CHILDREN_POSITION_FIELD_SIZE);
if (!DynamicPatriciaTrieWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) {
if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) {
AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos);
return false;
}

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
View File

@ -20,9 +20,9 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
@ -115,7 +115,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
BufferWithExtendableBuffer *const mTrieBuffer;
Ver4DictBuffers *const mBuffers;
const Ver4PatriciaTrieNodeReader *const mPtNodeReader;
DynamicPatriciaTrieReadingHelper mReadingHelper;
DynamicPtReadingHelper mReadingHelper;
Ver4BigramListPolicy *const mBigramPolicy;
Ver4ShortcutListPolicy *const mShortcutPolicy;
};

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View File

@ -21,7 +21,7 @@
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_vector.h"
#include "suggest/core/dictionary/unigram_property.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
#include "suggest/policyimpl/dictionary/utils/probability_utils.h"
@ -48,7 +48,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
if (!dicNode->hasChildren()) {
return;
}
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos());
while (!readingHelper.isEnd()) {
const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams();
@ -75,7 +75,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
const int ptNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const {
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodePos(ptNodePos);
return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount(
maxCodePointCount, outCodePoints, outUnigramProbability);
@ -83,7 +83,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) const {
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
}
@ -154,7 +154,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
mDictBuffer->getTailPosition());
return false;
}
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord,

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View File

@ -22,7 +22,7 @@
#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h"
@ -131,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
Ver4ShortcutListPolicy mShortcutPolicy;
Ver4PatriciaTrieNodeReader mNodeReader;
Ver4PatriciaTrieNodeWriter mNodeWriter;
DynamicPatriciaTrieUpdatingHelper mUpdatingHelper;
DynamicPtUpdatingHelper mUpdatingHelper;
Ver4PatriciaTrieWritingHelper mWritingHelper;
int mUnigramCount;
int mBigramCount;

12
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
View File

@ -88,9 +88,9 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader);
DynamicPtReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader);
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners
DynamicPtGcEventListeners
::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(
&ptNodeWriter);
@ -111,7 +111,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
}
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability
DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability
traversePolicyToUpdateBigramProbability(&ptNodeWriter);
if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(
&traversePolicyToUpdateBigramProbability)) {
@ -132,7 +132,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(),
buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers,
buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap);
if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
@ -170,10 +170,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
mBuffers->getShortcutDictContent())) {
return false;
}
DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(),
DynamicPtReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(),
&newPtNodeReader);
newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields
DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields
traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap);
if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner(
&traversePolicyToUpdateAllPositionFields)) {

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
View File

@ -18,7 +18,7 @@
#define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
namespace latinime {
@ -43,7 +43,7 @@ class Ver4PatriciaTrieWritingHelper {
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds
: public DynamicPatriciaTrieReadingHelper::TraversingEventListener {
: public DynamicPtReadingHelper::TraversingEventListener {
public:
TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds(
Ver4PatriciaTrieNodeWriter *const ptNodeWriter,

4
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -19,7 +19,7 @@
#include <cstdio>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
@ -51,7 +51,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
if (!DynamicPtWritingUtils::writeEmptyDictionary(
dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) {
AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
return false;

2
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View File

@ -46,8 +46,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
// same so we use them for both here.
if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_3) {
return VERSION_3;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4;
} else {

1
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View File

@ -31,7 +31,6 @@ class FormatUtils {
enum FORMAT_VERSION {
// These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2,
VERSION_3 = 3,
VERSION_4 = 400,
UNKNOWN_VERSION = -1
};