From 07e14126318f7661f76fdce421d723d64e7ea8de Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Tue, 10 Jun 2014 19:22:40 +0900 Subject: [PATCH] Purge dict reading code for 401 and create for 402. Change-Id: Ib0150b3306bb2db8aff443d1158fecc8464cbbdb --- .../latin/ExpandableBinaryDictionary.java | 3 +- native/jni/NativeFileList.mk | 10 +- .../dictionary/header/header_policy.h | 4 +- .../header/header_read_write_utils.cpp | 1 - .../backward/{v401 => v402}/Readme.txt | 0 .../bigram/ver4_bigram_list_policy.cpp | 12 +-- .../bigram/ver4_bigram_list_policy.h | 20 ++-- .../content/bigram_dict_content.cpp | 36 ++++---- .../content/bigram_dict_content.h | 37 +++++--- .../{v401 => v402}/content/bigram_entry.h | 12 +-- .../{v401 => v402}/content/dict_content.h | 10 +- .../content/probability_dict_content.cpp | 12 +-- .../content/probability_dict_content.h | 16 ++-- .../content/probability_entry.h | 12 +-- .../content/shortcut_dict_content.cpp | 6 +- .../content/shortcut_dict_content.h | 16 ++-- .../content/single_dict_content.h | 14 +-- .../content/sparse_table_dict_content.cpp | 6 +- .../content/sparse_table_dict_content.h | 14 +-- .../terminal_position_lookup_table.cpp | 8 +- .../content/terminal_position_lookup_table.h | 14 +-- .../shortcut/ver4_shortcut_list_policy.h | 14 +-- .../{v401 => v402}/ver4_dict_buffers.cpp | 7 +- .../{v401 => v402}/ver4_dict_buffers.h | 20 ++-- .../{v401 => v402}/ver4_dict_constants.cpp | 6 +- .../{v401 => v402}/ver4_dict_constants.h | 10 +- .../ver4_patricia_trie_node_reader.cpp | 12 +-- .../ver4_patricia_trie_node_reader.h | 18 ++-- .../ver4_patricia_trie_node_writer.cpp | 16 ++-- .../ver4_patricia_trie_node_writer.h | 20 ++-- .../ver4_patricia_trie_policy.cpp | 92 +++++++++++-------- .../ver4_patricia_trie_policy.h | 36 ++++---- .../ver4_patricia_trie_reading_utils.cpp | 6 +- .../ver4_patricia_trie_reading_utils.h | 14 +-- .../ver4_patricia_trie_writing_helper.cpp | 25 ++--- .../ver4_patricia_trie_writing_helper.h | 16 ++-- .../ver4_pt_node_array_reader.cpp | 6 +- .../ver4_pt_node_array_reader.h | 14 +-- ...y_structure_with_buffer_policy_factory.cpp | 29 +++--- .../utils/dict_file_writing_utils.cpp | 11 +-- .../dictionary/utils/format_utils.cpp | 18 +--- .../dictionary/utils/format_utils.h | 1 - 42 files changed, 333 insertions(+), 321 deletions(-) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/Readme.txt (100%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/bigram/ver4_bigram_list_policy.cpp (97%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/bigram/ver4_bigram_list_policy.h (89%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/bigram_dict_content.cpp (94%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/bigram_dict_content.h (78%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/bigram_entry.h (93%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/dict_content.h (85%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/probability_dict_content.cpp (96%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/probability_dict_content.h (85%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/probability_entry.h (91%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/shortcut_dict_content.cpp (98%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/shortcut_dict_content.h (90%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/single_dict_content.h (89%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/sparse_table_dict_content.cpp (94%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/sparse_table_dict_content.h (93%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/terminal_position_lookup_table.cpp (96%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/content/terminal_position_lookup_table.h (85%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/shortcut/ver4_shortcut_list_policy.h (93%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_dict_buffers.cpp (98%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_dict_buffers.h (90%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_dict_constants.cpp (97%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_dict_constants.h (93%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_node_reader.cpp (94%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_node_reader.h (88%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_node_writer.cpp (97%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_node_writer.h (94%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_policy.cpp (88%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_policy.h (88%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_reading_utils.cpp (92%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_reading_utils.h (82%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_writing_helper.cpp (95%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_patricia_trie_writing_helper.h (93%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_pt_node_array_reader.cpp (97%) rename native/jni/src/suggest/policyimpl/dictionary/structure/backward/{v401 => v402}/ver4_pt_node_array_reader.h (86%) diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 8664c09e4..0a5478c95 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -114,8 +114,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private boolean needsToMigrateDictionary(final int formatVersion) { // When we bump up the dictionary format version, the old version should be added to here // for supporting migration. Note that native code has to support reading such formats. - return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING - || formatVersion == FormatSpec.VERSION401; + return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING; } public boolean isValidDictionaryLocked() { diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk index 07a82a94f..2dd75c4f5 100644 --- a/native/jni/NativeFileList.mk +++ b/native/jni/NativeFileList.mk @@ -100,8 +100,8 @@ LATIN_IME_CORE_SRC_FILES := \ log_utils.cpp \ time_keeper.cpp) -LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \ - $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/, \ +LATIN_IME_CORE_SRC_FILES_BACKWARD_V402 := \ + $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/, \ ver4_dict_buffers.cpp \ ver4_dict_constants.cpp \ ver4_patricia_trie_node_reader.cpp \ @@ -110,16 +110,16 @@ LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \ ver4_patricia_trie_reading_utils.cpp \ ver4_patricia_trie_writing_helper.cpp \ ver4_pt_node_array_reader.cpp) \ - $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/content/, \ + $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/content/, \ bigram_dict_content.cpp \ probability_dict_content.cpp \ shortcut_dict_content.cpp \ sparse_table_dict_content.cpp \ terminal_position_lookup_table.cpp) \ - $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/bigram/, \ + $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/bigram/, \ ver4_bigram_list_policy.cpp) -LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V401) +LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V402) LATIN_IME_CORE_TEST_FILES := \ defines_test.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 75f4fef90..87cf0cd3b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -139,8 +139,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { switch (mDictFormatVersion) { case FormatUtils::VERSION_2: return FormatUtils::VERSION_2; - case FormatUtils::VERSION_401: - return FormatUtils::VERSION_401; case FormatUtils::VERSION_4_ONLY_FOR_TESTING: return FormatUtils::VERSION_4_ONLY_FOR_TESTING; case FormatUtils::VERSION_4: @@ -249,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { } bool supportsBeginningOfSentence() const { - return mDictFormatVersion > FormatUtils::VERSION_401; + return mDictFormatVersion >= FormatUtils::VERSION_4; } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index b13ad1879..a8f8f284b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -98,7 +98,6 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap; case FormatUtils::VERSION_2: // Version 2 dictionary writing is not supported. return false; - case FormatUtils::VERSION_401: case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: case FormatUtils::VERSION_4_DEV: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/Readme.txt b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/Readme.txt similarity index 100% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/Readme.txt rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/Readme.txt diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp similarity index 97% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp index 7ad072f09..3e8e059f2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp @@ -22,19 +22,19 @@ * suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h" #include "suggest/core/dictionary/property/bigram_property.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, int *const bigramEntryPos) const { @@ -285,6 +285,6 @@ bool Ver4BigramListPolicy::updateHasNextFlag(const bool hasNext, const int bigra return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, bigramEntryPos); } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h similarity index 89% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h index adf687bac..61623468e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h @@ -22,28 +22,28 @@ * suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H -#define LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H +#ifndef LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H +#define LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class BigramDictContent; -} // namespace v401 +} // namespace v402 } // namespace backward class BigramProperty; namespace backward { -namespace v401 { -} // namespace v401 +namespace v402 { +} // namespace v402 } // namespace backward class HeaderPolicy; namespace backward { -namespace v401 { +namespace v402 { class TerminalPositionLookupTable; class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { @@ -87,7 +87,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { const TerminalPositionLookupTable *const mTerminalPositionLookupTable; const HeaderPolicy *const mHeaderPolicy; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp similarity index 94% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp index 1e53ff94a..e2dd93c5e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp @@ -21,20 +21,22 @@ * suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( int *const bigramEntryPos) const { const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer(); - if (*bigramEntryPos < 0 || *bigramEntryPos >= bigramListBuffer->getTailPosition()) { - AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d", - *bigramEntryPos, bigramListBuffer->getTailPosition()); + const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize(); + if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) { + AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, " + "bufSize: %d", *bigramEntryPos, bigramEntryTailPos, + bigramListBuffer->getTailPosition()); ASSERT(false); return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY, Ver4DictConstants::NOT_A_TERMINAL_ID); @@ -47,8 +49,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( int level = 0; int count = 0; if (mHasHistoricalInfo) { - probability = bigramListBuffer->readUintAndAdvancePosition( - Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos); timestamp = bigramListBuffer->readUintAndAdvancePosition( Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos); level = bigramListBuffer->readUintAndAdvancePosition( @@ -56,7 +56,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( count = bigramListBuffer->readUintAndAdvancePosition( Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos); } else { - probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK; + probability = bigramListBuffer->readUintAndAdvancePosition( + Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos); } const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition( Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos); @@ -74,21 +75,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( bool BigramDictContent::writeBigramEntryAndAdvancePosition( const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) { BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer(); - const int bigramFlags = createAndGetBigramFlags( - mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(), - bigramEntryToWrite->hasNext()); + const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext()); if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags, Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) { AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags); return false; } if (mHasHistoricalInfo) { - if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(), - Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) { - AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos, - bigramEntryToWrite->getProbability()); - return false; - } const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo(); if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(), Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) { @@ -108,6 +101,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition( historicalInfo->getCount()); return false; } + } else { + if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(), + Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) { + AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos, + bigramEntryToWrite->getProbability()); + return false; + } } const int targetTerminalIdToWrite = (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ? @@ -219,6 +219,6 @@ bool BigramDictContent::runGCBigramList(const int bigramListPos, return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h similarity index 78% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h index f9c474b4a..b554e5676 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h @@ -21,18 +21,18 @@ * suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class BigramDictContent : public SparseTableDictContent { public: @@ -104,9 +104,22 @@ class BigramDictContent : public SparseTableDictContent { private: DISALLOW_COPY_AND_ASSIGN(BigramDictContent); - int createAndGetBigramFlags(const int probability, const bool hasNext) const { - return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK) - | (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0); + int createAndGetBigramFlags(const bool hasNext) const { + return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0; + } + + int getBigramEntrySize() const { + if (mHasHistoricalInfo) { + return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE + + Ver4DictConstants::TIME_STAMP_FIELD_SIZE + + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE + + Ver4DictConstants::WORD_COUNT_FIELD_SIZE + + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE; + } else { + return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE + + Ver4DictConstants::PROBABILITY_SIZE + + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE; + } } bool runGCBigramList(const int bigramListPos, @@ -116,7 +129,7 @@ class BigramDictContent : public SparseTableDictContent { bool mHasHistoricalInfo; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h similarity index 93% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h index 82c4b53a8..40968b4d8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h @@ -21,16 +21,16 @@ * suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h */ -#ifndef LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H -#define LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H +#ifndef LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H +#define LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/historical_info.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class BigramEntry { public: @@ -104,7 +104,7 @@ class BigramEntry { const HistoricalInfo mHistoricalInfo; const int mTargetTerminalId; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H */ +#endif /* LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h similarity index 85% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h index 39e29001c..0f2f25534 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h @@ -21,14 +21,14 @@ * suggest/policyimpl/dictionary/structure/v4/content/dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_DICT_CONTENT_H #include "defines.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class DictContent { public: @@ -41,7 +41,7 @@ class DictContent { private: DISALLOW_COPY_AND_ASSIGN(DictContent); }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp similarity index 96% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp index 337b97c05..c671647d4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp @@ -21,16 +21,16 @@ * suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const { if (terminalId < 0 || terminalId >= mSize) { @@ -166,6 +166,6 @@ bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilit return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h similarity index 85% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h index db3070994..3734797d4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h @@ -21,18 +21,18 @@ * suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class ProbabilityEntry; @@ -68,7 +68,7 @@ class ProbabilityDictContent : public SingleDictContent { bool mHasHistoricalInfo; int mSize; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h similarity index 91% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h index d341e7b07..8ccfa33dc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h @@ -21,16 +21,16 @@ * suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h */ -#ifndef LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H -#define LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H +#ifndef LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H +#define LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/historical_info.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class ProbabilityEntry { public: @@ -84,7 +84,7 @@ class ProbabilityEntry { const int mProbability; const HistoricalInfo mHistoricalInfo; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H */ +#endif /* LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp similarity index 98% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp index 3214807ad..56bc8b98d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp @@ -21,13 +21,13 @@ * suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePointCount, int *const outCodePoint, int *const outCodePointCount, int *const outProbability, @@ -194,6 +194,6 @@ int ShortcutDictContent::createAndGetShortcutFlags(const int probability, | (hasNext ? Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK : 0); } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h similarity index 90% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h index 75fd4f3b2..179cec5bb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h @@ -21,17 +21,17 @@ * suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class ShortcutDictContent : public SparseTableDictContent { public: @@ -95,7 +95,7 @@ class ShortcutDictContent : public SparseTableDictContent { int createAndGetShortcutFlags(const int probability, const bool hasNext) const; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h similarity index 89% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h index a519cd835..6433650b0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h @@ -21,19 +21,19 @@ * suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class SingleDictContent : public DictContent { public: @@ -80,7 +80,7 @@ class SingleDictContent : public DictContent { BufferWithExtendableBuffer mExpandableContentBuffer; const bool mIsValid; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp similarity index 94% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp index 638132c3d..7c9b4967a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp @@ -21,11 +21,11 @@ * suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { bool SparseTableDictContent::flush(const char *const dictPath, const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix, @@ -45,6 +45,6 @@ bool SparseTableDictContent::flush(const char *const dictPath, return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h similarity index 93% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h index b95de2eda..c7233edd3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h @@ -21,12 +21,12 @@ * suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h */ -#ifndef LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H -#define LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H +#ifndef LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H +#define LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" @@ -34,7 +34,7 @@ namespace latinime { namespace backward { -namespace v401 { +namespace v402 { // TODO: Support multiple contents. class SparseTableDictContent : public DictContent { @@ -116,7 +116,7 @@ class SparseTableDictContent : public DictContent { SparseTable mAddressLookupTable; const bool mIsValid; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H */ +#endif /* LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp similarity index 96% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp index ab8a3ae43..a9f841779 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp @@ -21,14 +21,14 @@ * suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const { if (terminalId < 0 || terminalId >= mSize) { @@ -106,6 +106,6 @@ bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminal return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h similarity index 85% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h index dbf0e6088..eadfe0faa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h @@ -21,18 +21,18 @@ * suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h */ -#ifndef LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H -#define LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H +#ifndef LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H +#define LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H #include #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class TerminalPositionLookupTable : public SingleDictContent { public: @@ -67,7 +67,7 @@ class TerminalPositionLookupTable : public SingleDictContent { int mSize; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif // LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H +#endif // LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h similarity index 93% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h index 6a4e83c0d..941fda748 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h @@ -22,18 +22,18 @@ * suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H -#define LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H +#ifndef LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H +#define LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H #include "defines.h" #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy { public: @@ -112,7 +112,7 @@ class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy { ShortcutDictContent *const mShortcutDictContent; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif // LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H +#endif // LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp similarity index 98% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp index 55ead01a9..93f192976 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp @@ -21,7 +21,7 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" #include #include @@ -33,7 +33,7 @@ namespace latinime { namespace backward { -namespace v401 { +namespace v402 { /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer, @@ -65,6 +65,7 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, return false; } } + umask(S_IWGRP | S_IWOTH); if (mkdir(tmpDirPath, S_IRWXU) == -1) { AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); return false; @@ -150,6 +151,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), mIsUpdatable(true) {} -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h similarity index 90% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h index 716ed931b..e775be52e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h @@ -21,24 +21,24 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H -#define LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H +#ifndef LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H +#define LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H #include #include "defines.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { class Ver4DictBuffers { public: @@ -146,7 +146,7 @@ class Ver4DictBuffers { ShortcutDictContent mShortcutDictContent; const int mIsUpdatable; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp similarity index 97% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp index 793b44ed4..81d85f495 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp @@ -21,11 +21,11 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { // These values MUST match the definitions in FormatSpec.java. const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie"; @@ -76,6 +76,6 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1; const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F; const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h similarity index 93% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h index 17afeb156..88ebd6a75 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h @@ -21,14 +21,14 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H -#define LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H +#ifndef LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H +#define LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H #include "defines.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { // TODO: Create PtConstants under the pt_common and move some constant values there. // Note that there are corresponding definitions in FormatSpec.java. @@ -78,7 +78,7 @@ class Ver4DictConstants { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants); }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp similarity index 94% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp index 80b51b292..82399f190 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp @@ -21,19 +21,19 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode( const int ptNodePos, const int siblingNodePos) const { @@ -104,6 +104,6 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce } } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h similarity index 88% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h index 0531b0a29..4032a67fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h @@ -21,8 +21,8 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H -#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H +#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H +#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H #include "defines.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" @@ -30,18 +30,18 @@ namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class BufferWithExtendableBuffer; namespace backward { -namespace v401 { -} // namespace v401 +namespace v402 { +} // namespace v402 } // namespace backward class HeaderPolicy; namespace backward { -namespace v401 { +namespace v402 { class ProbabilityDictContent; /* @@ -73,7 +73,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader { const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos, const int siblingNodePos) const; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp similarity index 97% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp index 8de6bacfc..4220a9561 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp @@ -21,24 +21,24 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h" #include "suggest/core/dictionary/property/unigram_property.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { const int Ver4PatriciaTrieNodeWriter::CHILDREN_POSITION_FIELD_SIZE = 3; @@ -424,6 +424,6 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h similarity index 94% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h index 7f1851d63..08226ea26 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h @@ -21,29 +21,29 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H -#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H +#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H +#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H #include "defines.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class BufferWithExtendableBuffer; namespace backward { -namespace v401 { -} // namespace v401 +namespace v402 { +} // namespace v402 } // namespace backward class HeaderPolicy; namespace backward { -namespace v401 { +namespace v402 { class Ver4BigramListPolicy; class Ver4DictBuffers; class Ver4PatriciaTrieNodeReader; @@ -139,7 +139,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp similarity index 88% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp index 0f60a898d..e571d8986 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp @@ -22,7 +22,7 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h" #include @@ -33,13 +33,13 @@ #include "suggest/core/dictionary/property/word_property.h" #include "suggest/core/session/prev_words_info.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { // Note that there are corresponding definitions in Java side in BinaryDictionaryTests and // BinaryDictionaryDecayingTests. @@ -70,13 +70,17 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d // valid terminal DicNode. isTerminal = ptNodeParams.getProbability() != NOT_A_PROBABILITY; } + readingHelper.readNextSiblingNode(ptNodeParams); + if (ptNodeParams.representsNonWordInfo()) { + // Skip PtNodes that represent non-word information. + continue; + } childDicNodes->pushLeavingChild(dicNode, ptNodeParams.getHeadPos(), ptNodeParams.getChildrenPos(), ptNodeParams.getProbability(), isTerminal, ptNodeParams.hasChildren(), ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord() /* isBlacklistedOrNotAWord */, ptNodeParams.getCodePointCount(), ptNodeParams.getCodePoints()); - readingHelper.readNextSiblingNode(ptNodeParams); } if (readingHelper.isError()) { mIsCorrupted = true; @@ -122,9 +126,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability, } else if (bigramProbability == NOT_A_PROBABILITY) { return ProbabilityUtils::backoff(unigramProbability); } else { - // bigramProbability is a bigram probability delta. - return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, - bigramProbability); + return bigramProbability; } } } @@ -189,9 +191,19 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); bool addedNewUnigram = false; - if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, + int codePointsToAdd[MAX_WORD_LENGTH]; + int codePointCountToAdd = length; + memmove(codePointsToAdd, word, sizeof(int) * length); + if (unigramProperty->representsBeginningOfSentence()) { + codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd, + codePointCountToAdd, MAX_WORD_LENGTH); + } + if (codePointCountToAdd <= 0) { + return false; + } + if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd, unigramProperty, &addedNewUnigram)) { - if (addedNewUnigram) { + if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) { mUnigramCount++; } if (unigramProperty->getShortcuts().size() > 0) { @@ -221,8 +233,6 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty) { - const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1); - const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1); if (!mBuffers->isUpdatable()) { AKLOGI("Warning: addNgramEntry() is called for non-updatable dictionary."); return false; @@ -232,15 +242,20 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI mDictBuffer->getTailPosition()); return false; } - if (length0 > MAX_WORD_LENGTH - || bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) { - AKLOGE("Either src word or target word is too long to insert the bigram to the dictionary. " - "length0: %d, length1: %d", length0, bigramProperty->getTargetCodePoints()->size()); + if (!prevWordsInfo->isValid()) { + AKLOGE("prev words info is not valid for adding n-gram entry to the dictionary."); return false; } - const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0, - false /* forceLowerCaseSearch */); - if (word0Pos == NOT_A_DICT_POS) { + if (bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) { + AKLOGE("The word is too long to insert the ngram to the dictionary. " + "length: %d", bigramProperty->getTargetCodePoints()->size()); + return false; + } + int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos, + false /* tryLowerCaseSearch */); + // TODO: Support N-gram. + if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) { return false; } const int word1Pos = getTerminalPtNodePositionOfWord( @@ -250,7 +265,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI return false; } bool addedNewBigram = false; - if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, bigramProperty, &addedNewBigram)) { + if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty, + &addedNewBigram)) { if (addedNewBigram) { mBigramCount++; } @@ -261,11 +277,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI } bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, - const int *const word1, const int length1) { - const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1); - const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1); + const int *const word, const int length) { if (!mBuffers->isUpdatable()) { - AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); + AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary."); return false; } if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { @@ -273,22 +287,26 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor mDictBuffer->getTailPosition()); return false; } - if (length0 > MAX_WORD_LENGTH || length1 > MAX_WORD_LENGTH) { - AKLOGE("Either src word or target word is too long to remove the bigram to from the " - "dictionary. length0: %d, length1: %d", length0, length1); + if (!prevWordsInfo->isValid()) { + AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary."); return false; } - const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0, + if (length > MAX_WORD_LENGTH) { + AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length); + } + int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos, + false /* tryLowerCaseSerch */); + // TODO: Support N-gram. + if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) { + return false; + } + const int wordPos = getTerminalPtNodePositionOfWord(word, length, false /* forceLowerCaseSearch */); - if (word0Pos == NOT_A_DICT_POS) { + if (wordPos == NOT_A_DICT_POS) { return false; } - const int word1Pos = getTerminalPtNodePositionOfWord(word1, length1, - false /* forceLowerCaseSearch */); - if (word1Pos == NOT_A_DICT_POS) { - return false; - } - if (mUpdatingHelper.removeBigramWords(word0Pos, word1Pos)) { + if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) { mBigramCount--; return true; } else { @@ -413,7 +431,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code const int probability = bigramEntry.hasHistoricalInfo() ? ForgettingCurveUtils::decodeProbability( bigramEntry.getHistoricalInfo(), mHeaderPolicy) : - getProbability(word1Probability, bigramEntry.getProbability()); + bigramEntry.getProbability(); bigrams.emplace_back(&word1, probability, historicalInfo->getTimeStamp(), historicalInfo->getLevel(), historicalInfo->getCount()); @@ -436,7 +454,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code shortcuts.emplace_back(&target, shortcutProbability); } } - const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */, + const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(), ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(), historicalInfo->getTimeStamp(), historicalInfo->getLevel(), historicalInfo->getCount(), &shortcuts); @@ -479,6 +497,6 @@ int Ver4PatriciaTriePolicy::getNextWordAndNextToken(const int token, int *const return nextToken; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h similarity index 88% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h index b064aaf33..e323652d4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h @@ -22,8 +22,8 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H -#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H +#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H +#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H #include @@ -31,29 +31,29 @@ #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class DicNode; namespace backward { -namespace v401 { -} // namespace v401 +namespace v402 { +} // namespace v402 } // namespace backward class DicNodeVector; namespace backward { -namespace v401 { +namespace v402 { class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: @@ -114,8 +114,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo, const BigramProperty *const bigramProperty); - bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word, - const int length); + bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1, + const int length1); bool flush(const char *const filePath); @@ -162,7 +162,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { std::vector mTerminalPtNodePositionsForIteratingWords; mutable bool mIsCorrupted; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif // LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H +#endif // LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp similarity index 92% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp index 6cc36fbef..80d531198 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp @@ -21,19 +21,19 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { /* static */ int Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition( const uint8_t *const buffer, int *pos) { return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos); } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h similarity index 82% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h index 7417c261e..3579c26d6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h @@ -21,8 +21,8 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H -#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H +#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H +#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H #include @@ -30,13 +30,13 @@ namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class BufferWithExtendableBuffer; namespace backward { -namespace v401 { +namespace v402 { class Ver4PatriciaTrieReadingUtils { public: @@ -46,7 +46,7 @@ class Ver4PatriciaTrieReadingUtils { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils); }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp similarity index 95% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp index 10f27beb7..99eed0f67 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp @@ -21,26 +21,26 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h" #include #include #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { bool Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath, const int unigramCount, const int bigramCount) const { @@ -222,13 +222,16 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams( // Delete unigrams. while (static_cast(priorityQueue.size()) > maxUnigramCount) { const int ptNodePos = priorityQueue.top().getDictPos(); + priorityQueue.pop(); const PtNodeParams ptNodeParams = ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos); + if (ptNodeParams.representsNonWordInfo()) { + continue; + } if (!ptNodeWriter->markPtNodeAsWillBecomeNonTerminal(&ptNodeParams)) { AKLOGE("Cannot mark PtNode as willBecomeNonterminal. PtNode pos: %d", ptNodePos); return false; } - priorityQueue.pop(); } return true; } @@ -296,6 +299,6 @@ bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllPtNodeFlagsAndTermi return mPtNodeWriter->updatePtNodeHasBigramsAndShortcutTargetsFlags(ptNodeParams); } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h similarity index 93% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h index be44aaa33..9034ee656 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h @@ -21,22 +21,22 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H -#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H +#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H +#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H #include "defines.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class HeaderPolicy; namespace backward { -namespace v401 { +namespace v402 { class Ver4DictBuffers; class Ver4PatriciaTrieNodeReader; class Ver4PatriciaTrieNodeWriter; @@ -133,8 +133,8 @@ class Ver4PatriciaTrieWritingHelper { Ver4DictBuffers *const mBuffers; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp similarity index 97% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.cpp rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp index 33e4e55e2..537a6d420 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp @@ -21,7 +21,7 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.cpp */ -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h" @@ -29,7 +29,7 @@ namespace latinime { namespace backward { -namespace v401 { +namespace v402 { bool Ver4PtNodeArrayReader::readPtNodeArrayInfoAndReturnIfValid(const int ptNodeArrayPos, int *const outPtNodeCount, int *const outFirstPtNodePos) const { @@ -85,6 +85,6 @@ bool Ver4PtNodeArrayReader::readForwardLinkAndReturnIfValid(const int forwordLin return true; } -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h similarity index 86% rename from native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h rename to native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h index 3a7eefa44..4f8056801 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h @@ -21,21 +21,21 @@ * suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h */ -#ifndef LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H -#define LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H +#ifndef LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H +#define LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H #include "defines.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_array_reader.h" namespace latinime { namespace backward { -namespace v401 { +namespace v402 { -} // namespace v401 +} // namespace v402 } // namespace backward class BufferWithExtendableBuffer; namespace backward { -namespace v401 { +namespace v402 { class Ver4PtNodeArrayReader : public PtNodeArrayReader { public: @@ -51,7 +51,7 @@ class Ver4PtNodeArrayReader : public PtNodeArrayReader { const BufferWithExtendableBuffer *const mBuffer; }; -} // namespace v401 +} // namespace v402 } // namespace backward } // namespace latinime -#endif /* LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H */ +#endif /* LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index 93e330a2a..e4b5fa267 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -19,9 +19,9 @@ #include #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" @@ -57,14 +57,13 @@ namespace latinime { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion); switch (dictFormatVersion) { - case FormatUtils::VERSION_401: { - return newPolicyForOnMemoryV4Dict( + case FormatUtils::VERSION_4: { + return newPolicyForOnMemoryV4Dict( dictFormatVersion, locale, attributeMap); } - case FormatUtils::VERSION_4: case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_DEV: { return newPolicyForOnMemoryV4Dict( + case FormatUtils::VERSION_4: { + return newPolicyForV4Dict( headerFilePath, formatVersion, std::move(mmappedBuffer)); } - case FormatUtils::VERSION_4: case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_DEV: { return newPolicyForV4Dict #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -41,12 +41,11 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = TimeKeeper::setCurrentTime(); const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion); switch (formatVersion) { - case FormatUtils::VERSION_401: - return createEmptyV4DictFile( - filePath, localeAsCodePointVector, attributeMap, formatVersion); case FormatUtils::VERSION_4: + return createEmptyV4DictFile( + filePath, localeAsCodePointVector, attributeMap, formatVersion); case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_DEV: return createEmptyV4DictFile