Purge dict reading code for 401 and create for 402.

Change-Id: Ib0150b3306bb2db8aff443d1158fecc8464cbbdb

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -114,8 +114,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     private boolean needsToMigrateDictionary(final int formatVersion) {
         // When we bump up the dictionary format version, the old version should be added to here
         // for supporting migration. Note that native code has to support reading such formats.
-        return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
+        return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING;
-                || formatVersion == FormatSpec.VERSION401;
     }
 
     public boolean isValidDictionaryLocked() {

native/jni/NativeFileList.mk

@@ -100,8 +100,8 @@ LATIN_IME_CORE_SRC_FILES := \
         log_utils.cpp \
         time_keeper.cpp)
 
-LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
+LATIN_IME_CORE_SRC_FILES_BACKWARD_V402 := \
-    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/, \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/, \
         ver4_dict_buffers.cpp \
         ver4_dict_constants.cpp \
         ver4_patricia_trie_node_reader.cpp \
@@ -110,16 +110,16 @@ LATIN_IME_CORE_SRC_FILES_BACKWARD_V401 := \
         ver4_patricia_trie_reading_utils.cpp \
         ver4_patricia_trie_writing_helper.cpp \
         ver4_pt_node_array_reader.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/content/, \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/content/, \
         bigram_dict_content.cpp \
         probability_dict_content.cpp \
         shortcut_dict_content.cpp \
         sparse_table_dict_content.cpp \
         terminal_position_lookup_table.cpp) \
-    $(addprefix suggest/policyimpl/dictionary/structure/backward/v401/bigram/, \
+    $(addprefix suggest/policyimpl/dictionary/structure/backward/v402/bigram/, \
         ver4_bigram_list_policy.cpp)
 
-LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V401)
+LATIN_IME_CORE_SRC_FILES += $(LATIN_IME_CORE_SRC_FILES_BACKWARD_V402)
 
 LATIN_IME_CORE_TEST_FILES := \
     defines_test.cpp \

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

@@ -139,8 +139,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
         switch (mDictFormatVersion) {
             case FormatUtils::VERSION_2:
                 return FormatUtils::VERSION_2;
-            case FormatUtils::VERSION_401:
-                return FormatUtils::VERSION_401;
             case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
                 return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
             case FormatUtils::VERSION_4:
@@ -249,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     }
 
     bool supportsBeginningOfSentence() const {
-        return mDictFormatVersion > FormatUtils::VERSION_401;
+        return mDictFormatVersion >= FormatUtils::VERSION_4;
     }
 
  private:

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp

@@ -98,7 +98,6 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
         case FormatUtils::VERSION_2:
             // Version 2 dictionary writing is not supported.
             return false;
-        case FormatUtils::VERSION_401:
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4:
         case FormatUtils::VERSION_4_DEV:

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.cpp

@@ -22,19 +22,19 @@
  *   suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
 
 #include "suggest/core/dictionary/property/bigram_property.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,
         bool *const outHasNext, int *const bigramEntryPos) const {
@@ -285,6 +285,6 @@ bool Ver4BigramListPolicy::updateHasNextFlag(const bool hasNext, const int bigra
     return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, bigramEntryPos);
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h

@@ -22,28 +22,28 @@
  *   suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
+#ifndef LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
-#define LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H
+#define LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H
 
 #include "defines.h"
 #include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class BigramDictContent;
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class BigramProperty;
 namespace backward {
-namespace v401 {
+namespace v402 {
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class HeaderPolicy;
 namespace backward {
-namespace v401 {
+namespace v402 {
 class TerminalPositionLookupTable;
 
 class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
@@ -87,7 +87,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
     const TerminalPositionLookupTable *const mTerminalPositionLookupTable;
     const HeaderPolicy *const mHeaderPolicy;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_BIGRAM_LIST_POLICY_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_BIGRAM_LIST_POLICY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.cpp

@@ -21,20 +21,22 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
 
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
         int *const bigramEntryPos) const {
     const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
-    if (*bigramEntryPos < 0 || *bigramEntryPos >=  bigramListBuffer->getTailPosition()) {
+    const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize();
-        AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d",
+    if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) {
-                *bigramEntryPos, bigramListBuffer->getTailPosition());
+        AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, "
+                "bufSize: %d", *bigramEntryPos, bigramEntryTailPos,
+                        bigramListBuffer->getTailPosition());
         ASSERT(false);
         return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
                 Ver4DictConstants::NOT_A_TERMINAL_ID);
@@ -47,8 +49,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
     int level = 0;
     int count = 0;
     if (mHasHistoricalInfo) {
-        probability = bigramListBuffer->readUintAndAdvancePosition(
-                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
         timestamp = bigramListBuffer->readUintAndAdvancePosition(
                 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
         level = bigramListBuffer->readUintAndAdvancePosition(
@@ -56,7 +56,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
         count = bigramListBuffer->readUintAndAdvancePosition(
                 Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
     } else {
-        probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
+        probability = bigramListBuffer->readUintAndAdvancePosition(
+                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
     }
     const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
             Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@@ -74,21 +75,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
 bool BigramDictContent::writeBigramEntryAndAdvancePosition(
         const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
     BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
-    const int bigramFlags = createAndGetBigramFlags(
+    const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
-            mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
-            bigramEntryToWrite->hasNext());
     if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
             Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
         AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
         return false;
     }
     if (mHasHistoricalInfo) {
-        if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
-                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
-            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
-                    bigramEntryToWrite->getProbability());
-            return false;
-        }
         const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
         if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
                 Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@@ -108,6 +101,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
                     historicalInfo->getCount());
             return false;
         }
+    } else {
+        if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
+                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
+            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
+                    bigramEntryToWrite->getProbability());
+            return false;
+        }
     }
     const int targetTerminalIdToWrite =
             (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
@@ -219,6 +219,6 @@ bool BigramDictContent::runGCBigramList(const int bigramListPos,
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h

@@ -21,18 +21,18 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class BigramDictContent : public SparseTableDictContent {
  public:
@@ -104,9 +104,22 @@ class BigramDictContent : public SparseTableDictContent {
  private:
     DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
 
-    int createAndGetBigramFlags(const int probability, const bool hasNext) const {
+    int createAndGetBigramFlags(const bool hasNext) const {
-        return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
+        return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
-                | (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
+    }
+
+    int getBigramEntrySize() const {
+        if (mHasHistoricalInfo) {
+            return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+                    + Ver4DictConstants::TIME_STAMP_FIELD_SIZE
+                    + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
+                    + Ver4DictConstants::WORD_COUNT_FIELD_SIZE
+                    + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
+        } else {
+            return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
+                    + Ver4DictConstants::PROBABILITY_SIZE
+                    + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
+        }
     }
 
     bool runGCBigramList(const int bigramListPos,
@@ -116,7 +129,7 @@ class BigramDictContent : public SparseTableDictContent {
 
     bool mHasHistoricalInfo;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_BIGRAM_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_BIGRAM_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_entry.h

@@ -21,16 +21,16 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
+#ifndef LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
-#define LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H
+#define LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/historical_info.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class BigramEntry {
  public:
@@ -104,7 +104,7 @@ class BigramEntry {
     const HistoricalInfo mHistoricalInfo;
     const int mTargetTerminalId;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_BIGRAM_ENTRY_H */
+#endif /* LATINIME_BACKWARD_V402_BIGRAM_ENTRY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h

@@ -21,14 +21,14 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_DICT_CONTENT_H
 
 #include "defines.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class DictContent {
  public:
@@ -41,7 +41,7 @@ class DictContent {
  private:
     DISALLOW_COPY_AND_ASSIGN(DictContent);
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp

@@ -21,16 +21,16 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 const ProbabilityEntry ProbabilityDictContent::getProbabilityEntry(const int terminalId) const {
     if (terminalId < 0 || terminalId >= mSize) {
@@ -166,6 +166,6 @@ bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilit
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h

@@ -21,18 +21,18 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class ProbabilityEntry;
 
@@ -68,7 +68,7 @@ class ProbabilityDictContent : public SingleDictContent {
     bool mHasHistoricalInfo;
     int mSize;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_PROBABILITY_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_PROBABILITY_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h

@@ -21,16 +21,16 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
+#ifndef LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
-#define LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H
+#define LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/historical_info.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class ProbabilityEntry {
  public:
@@ -84,7 +84,7 @@ class ProbabilityEntry {
     const int mProbability;
     const HistoricalInfo mHistoricalInfo;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_PROBABILITY_ENTRY_H */
+#endif /* LATINIME_BACKWARD_V402_PROBABILITY_ENTRY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.cpp

@@ -21,13 +21,13 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
 
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
         int *const outCodePoint, int *const outCodePointCount, int *const outProbability,
@@ -194,6 +194,6 @@ int ShortcutDictContent::createAndGetShortcutFlags(const int probability,
             | (hasNext ? Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK : 0);
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h

@@ -21,17 +21,17 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class ShortcutDictContent : public SparseTableDictContent {
  public:
@@ -95,7 +95,7 @@ class ShortcutDictContent : public SparseTableDictContent {
 
     int createAndGetShortcutFlags(const int probability, const bool hasNext) const;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_SHORTCUT_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_SHORTCUT_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h

@@ -21,19 +21,19 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class SingleDictContent : public DictContent {
  public:
@@ -80,7 +80,7 @@ class SingleDictContent : public DictContent {
     BufferWithExtendableBuffer mExpandableContentBuffer;
     const bool mIsValid;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_SINGLE_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_SINGLE_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.cpp

@@ -21,11 +21,11 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/sparse_table_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 bool SparseTableDictContent::flush(const char *const dictPath,
         const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
@@ -45,6 +45,6 @@ bool SparseTableDictContent::flush(const char *const dictPath,
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/sparse_table_dict_content.h

@@ -21,12 +21,12 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
+#ifndef LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
-#define LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H
+#define LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
@@ -34,7 +34,7 @@
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 // TODO: Support multiple contents.
 class SparseTableDictContent : public DictContent {
@@ -116,7 +116,7 @@ class SparseTableDictContent : public DictContent {
     SparseTable mAddressLookupTable;
     const bool mIsValid;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_SPARSE_TABLE_DICT_CONTENT_H */
+#endif /* LATINIME_BACKWARD_V402_SPARSE_TABLE_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.cpp

@@ -21,14 +21,14 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const {
     if (terminalId < 0 || terminalId >= mSize) {
@@ -106,6 +106,6 @@ bool TerminalPositionLookupTable::runGCTerminalIds(TerminalIdMap *const terminal
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h

@@ -21,18 +21,18 @@
  *   suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
+#ifndef LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
-#define LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
+#define LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H
 
 #include <unordered_map>
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/single_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/single_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class TerminalPositionLookupTable : public SingleDictContent {
  public:
@@ -67,7 +67,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
 
     int mSize;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif // LATINIME_BACKWARD_V401_TERMINAL_POSITION_LOOKUP_TABLE_H
+#endif // LATINIME_BACKWARD_V402_TERMINAL_POSITION_LOOKUP_TABLE_H

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h

@@ -22,18 +22,18 @@
  *   suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
+#ifndef LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
-#define LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
+#define LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H
 
 #include "defines.h"
 #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
  public:
@@ -112,7 +112,7 @@ class Ver4ShortcutListPolicy : public DictionaryShortcutsStructurePolicy {
 
     ShortcutDictContent *const mShortcutDictContent;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif // LATINIME_BACKWARD_V401_VER4_SHORTCUT_LIST_POLICY_H
+#endif // LATINIME_BACKWARD_V402_VER4_SHORTCUT_LIST_POLICY_H

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.cpp

@@ -21,7 +21,7 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
 
 #include <cerrno>
 #include <cstring>
@@ -33,7 +33,7 @@
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
         const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
@@ -65,6 +65,7 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
             return false;
         }
     }
+    umask(S_IWGRP | S_IWOTH);
     if (mkdir(tmpDirPath, S_IRWXU) == -1) {
         AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
         return false;
@@ -150,6 +151,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i
           mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
           mIsUpdatable(true) {}
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h

@@ -21,24 +21,24 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
+#ifndef LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
-#define LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H
+#define LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H
 
 #include <memory>
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/bigram_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/bigram_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/shortcut_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/shortcut_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class Ver4DictBuffers {
  public:
@@ -146,7 +146,7 @@ class Ver4DictBuffers {
     ShortcutDictContent mShortcutDictContent;
     const int mIsUpdatable;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_DICT_BUFFER_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_DICT_BUFFER_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.cpp

@@ -21,11 +21,11 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 // These values MUST match the definitions in FormatSpec.java.
 const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
@@ -76,6 +76,6 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
 const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
 const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h

@@ -21,14 +21,14 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
+#ifndef LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
-#define LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H
+#define LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H
 
 #include "defines.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 // TODO: Create PtConstants under the pt_common and move some constant values there.
 // Note that there are corresponding definitions in FormatSpec.java.
@@ -78,7 +78,7 @@ class Ver4DictConstants {
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_DICT_CONSTANTS_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_DICT_CONSTANTS_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.cpp

@@ -21,19 +21,19 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
 
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_dict_content.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode(
         const int ptNodePos, const int siblingNodePos) const {
@@ -104,6 +104,6 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce
     }
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h

@@ -21,8 +21,8 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
-#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H
+#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
@@ -30,18 +30,18 @@
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class BufferWithExtendableBuffer;
 namespace backward {
-namespace v401 {
+namespace v402 {
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class HeaderPolicy;
 namespace backward {
-namespace v401 {
+namespace v402 {
 class ProbabilityDictContent;
 
 /*
@@ -73,7 +73,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader {
     const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos,
             const int siblingNodePos) const;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_READER_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_READER_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.cpp

@@ -21,24 +21,24 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
 
 #include "suggest/core/dictionary/property/unigram_property.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 const int Ver4PatriciaTrieNodeWriter::CHILDREN_POSITION_FIELD_SIZE = 3;
 
@@ -424,6 +424,6 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos,
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h

@@ -21,29 +21,29 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
-#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H
+#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/probability_entry.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/probability_entry.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class BufferWithExtendableBuffer;
 namespace backward {
-namespace v401 {
+namespace v402 {
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class HeaderPolicy;
 namespace backward {
-namespace v401 {
+namespace v402 {
 class Ver4BigramListPolicy;
 class Ver4DictBuffers;
 class Ver4PatriciaTrieNodeReader;
@@ -139,7 +139,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     Ver4BigramListPolicy *const mBigramPolicy;
     Ver4ShortcutListPolicy *const mShortcutPolicy;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_NODE_WRITER_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_NODE_WRITER_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp

@@ -22,7 +22,7 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
 
 #include <vector>
 
@@ -33,13 +33,13 @@
 #include "suggest/core/dictionary/property/word_property.h"
 #include "suggest/core/session/prev_words_info.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 #include "suggest/policyimpl/dictionary/utils/probability_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 // Note that there are corresponding definitions in Java side in BinaryDictionaryTests and
 // BinaryDictionaryDecayingTests.
@@ -70,13 +70,17 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
             // valid terminal DicNode.
             isTerminal = ptNodeParams.getProbability() != NOT_A_PROBABILITY;
         }
+        readingHelper.readNextSiblingNode(ptNodeParams);
+        if (ptNodeParams.representsNonWordInfo()) {
+            // Skip PtNodes that represent non-word information.
+            continue;
+        }
         childDicNodes->pushLeavingChild(dicNode, ptNodeParams.getHeadPos(),
                 ptNodeParams.getChildrenPos(), ptNodeParams.getProbability(), isTerminal,
                 ptNodeParams.hasChildren(),
                 ptNodeParams.isBlacklisted()
                         || ptNodeParams.isNotAWord() /* isBlacklistedOrNotAWord */,
                 ptNodeParams.getCodePointCount(), ptNodeParams.getCodePoints());
-        readingHelper.readNextSiblingNode(ptNodeParams);
     }
     if (readingHelper.isError()) {
         mIsCorrupted = true;
@@ -122,9 +126,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
         } else if (bigramProbability == NOT_A_PROBABILITY) {
             return ProbabilityUtils::backoff(unigramProbability);
         } else {
-            // bigramProbability is a bigram probability delta.
+            return bigramProbability;
-            return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
-                    bigramProbability);
         }
     }
 }
@@ -189,9 +191,19 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
     DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
     readingHelper.initWithPtNodeArrayPos(getRootPosition());
     bool addedNewUnigram = false;
-    if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length,
+    int codePointsToAdd[MAX_WORD_LENGTH];
+    int codePointCountToAdd = length;
+    memmove(codePointsToAdd, word, sizeof(int) * length);
+    if (unigramProperty->representsBeginningOfSentence()) {
+        codePointCountToAdd = CharUtils::attachBeginningOfSentenceMarker(codePointsToAdd,
+                codePointCountToAdd, MAX_WORD_LENGTH);
+    }
+    if (codePointCountToAdd <= 0) {
+        return false;
+    }
+    if (mUpdatingHelper.addUnigramWord(&readingHelper, codePointsToAdd, codePointCountToAdd,
             unigramProperty, &addedNewUnigram)) {
-        if (addedNewUnigram) {
+        if (addedNewUnigram && !unigramProperty->representsBeginningOfSentence()) {
             mUnigramCount++;
         }
         if (unigramProperty->getShortcuts().size() > 0) {
@@ -221,8 +233,6 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
 
 bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
         const BigramProperty *const bigramProperty) {
-    const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
-    const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
     if (!mBuffers->isUpdatable()) {
         AKLOGI("Warning: addNgramEntry() is called for non-updatable dictionary.");
         return false;
@@ -232,15 +242,20 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
                 mDictBuffer->getTailPosition());
         return false;
     }
-    if (length0 > MAX_WORD_LENGTH
+    if (!prevWordsInfo->isValid()) {
-            || bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
+        AKLOGE("prev words info is not valid for adding n-gram entry to the dictionary.");
-        AKLOGE("Either src word or target word is too long to insert the bigram to the dictionary. "
-                "length0: %d, length1: %d", length0, bigramProperty->getTargetCodePoints()->size());
         return false;
     }
-    const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
+    if (bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) {
-            false /* forceLowerCaseSearch */);
+        AKLOGE("The word is too long to insert the ngram to the dictionary. "
-    if (word0Pos == NOT_A_DICT_POS) {
+                "length: %d", bigramProperty->getTargetCodePoints()->size());
+        return false;
+    }
+    int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
+    prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
+            false /* tryLowerCaseSearch */);
+    // TODO: Support N-gram.
+    if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
         return false;
     }
     const int word1Pos = getTerminalPtNodePositionOfWord(
@@ -250,7 +265,8 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
         return false;
     }
     bool addedNewBigram = false;
-    if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, bigramProperty, &addedNewBigram)) {
+    if (mUpdatingHelper.addBigramWords(prevWordsPtNodePos[0], word1Pos, bigramProperty,
+            &addedNewBigram)) {
         if (addedNewBigram) {
             mBigramCount++;
         }
@@ -261,11 +277,9 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI
 }
 
 bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
-        const int *const word1, const int length1) {
+        const int *const word, const int length) {
-    const int length0 = prevWordsInfo->getNthPrevWordCodePointCount(1);
-    const int *word0 = prevWordsInfo->getNthPrevWordCodePoints(1);
     if (!mBuffers->isUpdatable()) {
-        AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
+        AKLOGI("Warning: removeNgramEntry() is called for non-updatable dictionary.");
         return false;
     }
     if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
@@ -273,22 +287,26 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
                 mDictBuffer->getTailPosition());
         return false;
     }
-    if (length0 > MAX_WORD_LENGTH || length1 > MAX_WORD_LENGTH) {
+    if (!prevWordsInfo->isValid()) {
-        AKLOGE("Either src word or target word is too long to remove the bigram to from the "
+        AKLOGE("prev words info is not valid for removing n-gram entry form the dictionary.");
-                "dictionary. length0: %d, length1: %d", length0, length1);
         return false;
     }
-    const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
+    if (length > MAX_WORD_LENGTH) {
+        AKLOGE("word is too long to remove n-gram entry form the dictionary. length: %d", length);
+    }
+    int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];
+    prevWordsInfo->getPrevWordsTerminalPtNodePos(this, prevWordsPtNodePos,
+            false /* tryLowerCaseSerch */);
+    // TODO: Support N-gram.
+    if (prevWordsPtNodePos[0] == NOT_A_DICT_POS) {
+        return false;
+    }
+    const int wordPos = getTerminalPtNodePositionOfWord(word, length,
             false /* forceLowerCaseSearch */);
-    if (word0Pos == NOT_A_DICT_POS) {
+    if (wordPos == NOT_A_DICT_POS) {
         return false;
     }
-    const int word1Pos = getTerminalPtNodePositionOfWord(word1, length1,
+    if (mUpdatingHelper.removeBigramWords(prevWordsPtNodePos[0], wordPos)) {
-            false /* forceLowerCaseSearch */);
-    if (word1Pos == NOT_A_DICT_POS) {
-        return false;
-    }
-    if (mUpdatingHelper.removeBigramWords(word0Pos, word1Pos)) {
         mBigramCount--;
         return true;
     } else {
@@ -413,7 +431,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
             const int probability = bigramEntry.hasHistoricalInfo() ?
                     ForgettingCurveUtils::decodeProbability(
                             bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
-                    getProbability(word1Probability, bigramEntry.getProbability());
+                    bigramEntry.getProbability();
             bigrams.emplace_back(&word1, probability,
                     historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
                     historicalInfo->getCount());
@@ -436,7 +454,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
             shortcuts.emplace_back(&target, shortcutProbability);
         }
     }
-    const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */,
+    const UnigramProperty unigramProperty(ptNodeParams.representsBeginningOfSentence(),
             ptNodeParams.isNotAWord(), ptNodeParams.isBlacklisted(), ptNodeParams.getProbability(),
             historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
             historicalInfo->getCount(), &shortcuts);
@@ -479,6 +497,6 @@ int Ver4PatriciaTriePolicy::getNextWordAndNextToken(const int token, int *const
     return nextToken;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h

@@ -22,8 +22,8 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
-#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
+#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H
 
 #include <vector>
 
@@ -31,29 +31,29 @@
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class DicNode;
 namespace backward {
-namespace v401 {
+namespace v402 {
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class DicNodeVector;
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
@@ -114,8 +114,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
             const BigramProperty *const bigramProperty);
 
-    bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
+    bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
-            const int length);
+            const int length1);
 
     bool flush(const char *const filePath);
 
@@ -162,7 +162,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     std::vector<int> mTerminalPtNodePositionsForIteratingWords;
     mutable bool mIsCorrupted;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif // LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_POLICY_H
+#endif // LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_POLICY_H

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.cpp

@@ -21,19 +21,19 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_reading_utils.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h"
 
 #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 /* static */ int Ver4PatriciaTrieReadingUtils::getTerminalIdAndAdvancePosition(
         const uint8_t *const buffer, int *pos) {
     return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_reading_utils.h

@@ -21,8 +21,8 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
-#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H
+#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H
 
 #include <cstdint>
 
@@ -30,13 +30,13 @@
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class BufferWithExtendableBuffer;
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class Ver4PatriciaTrieReadingUtils {
  public:
@@ -46,7 +46,7 @@ class Ver4PatriciaTrieReadingUtils {
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieReadingUtils);
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_READING_UTILS_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_READING_UTILS_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp

@@ -21,26 +21,26 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_writing_helper.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h"
 
 #include <cstring>
 #include <queue>
 
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/bigram/ver4_bigram_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/bigram/ver4_bigram_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/shortcut/ver4_shortcut_list_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/shortcut/ver4_shortcut_list_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_node_writer.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_writer.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/file_utils.h"
 #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 bool Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath,
         const int unigramCount, const int bigramCount) const {
@@ -222,13 +222,16 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
     // Delete unigrams.
     while (static_cast<int>(priorityQueue.size()) > maxUnigramCount) {
         const int ptNodePos = priorityQueue.top().getDictPos();
+        priorityQueue.pop();
         const PtNodeParams ptNodeParams =
                 ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+        if (ptNodeParams.representsNonWordInfo()) {
+            continue;
+        }
         if (!ptNodeWriter->markPtNodeAsWillBecomeNonTerminal(&ptNodeParams)) {
             AKLOGE("Cannot mark PtNode as willBecomeNonterminal. PtNode pos: %d", ptNodePos);
             return false;
         }
-        priorityQueue.pop();
     }
     return true;
 }
@@ -296,6 +299,6 @@ bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllPtNodeFlagsAndTermi
     return mPtNodeWriter->updatePtNodeHasBigramsAndShortcutTargetsFlags(ptNodeParams);
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.h

@@ -21,22 +21,22 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
-#define LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H
+#define LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/content/terminal_position_lookup_table.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/content/terminal_position_lookup_table.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class HeaderPolicy;
 namespace backward {
-namespace v401 {
+namespace v402 {
 class Ver4DictBuffers;
 class Ver4PatriciaTrieNodeReader;
 class Ver4PatriciaTrieNodeWriter;
@@ -133,8 +133,8 @@ class Ver4PatriciaTrieWritingHelper {
 
     Ver4DictBuffers *const mBuffers;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
 
-#endif /* LATINIME_BACKWARD_V401_VER4_PATRICIA_TRIE_WRITING_HELPER_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_PATRICIA_TRIE_WRITING_HELPER_H */

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.cpp

@@ -21,7 +21,7 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.cpp
  */
 
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_pt_node_array_reader.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h"
 
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
@@ -29,7 +29,7 @@
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 bool Ver4PtNodeArrayReader::readPtNodeArrayInfoAndReturnIfValid(const int ptNodeArrayPos,
         int *const outPtNodeCount, int *const outFirstPtNodePos) const {
@@ -85,6 +85,6 @@ bool Ver4PtNodeArrayReader::readForwardLinkAndReturnIfValid(const int forwordLin
     return true;
 }
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_pt_node_array_reader.h

@@ -21,21 +21,21 @@
  *   suggest/policyimpl/dictionary/structure/v4/ver4_pt_node_array_reader.h
  */
 
-#ifndef LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
+#ifndef LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
-#define LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H
+#define LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_array_reader.h"
 
 namespace latinime {
 namespace backward {
-namespace v401 {
+namespace v402 {
 
-} // namespace v401
+} // namespace v402
 } // namespace backward
 class BufferWithExtendableBuffer;
 namespace backward {
-namespace v401 {
+namespace v402 {
 
 class Ver4PtNodeArrayReader : public PtNodeArrayReader {
  public:
@@ -51,7 +51,7 @@ class Ver4PtNodeArrayReader : public PtNodeArrayReader {
 
     const BufferWithExtendableBuffer *const mBuffer;
 };
-} // namespace v401
+} // namespace v402
 } // namespace backward
 } // namespace latinime
-#endif /* LATINIME_BACKWARD_V401_VER4_PT_NODE_ARRAY_READER_H */
+#endif /* LATINIME_BACKWARD_V402_VER4_PT_NODE_ARRAY_READER_H */

native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp

@@ -19,9 +19,9 @@
 #include <climits>
 
 #include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
@@ -57,14 +57,13 @@ namespace latinime {
                 const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
     FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
     switch (dictFormatVersion) {
-        case FormatUtils::VERSION_401: {
+        case FormatUtils::VERSION_4: {
-            return newPolicyForOnMemoryV4Dict<backward::v401::Ver4DictConstants,
+            return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
-                    backward::v401::Ver4DictBuffers,
+                    backward::v402::Ver4DictBuffers,
-                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
+                    backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
-                    backward::v401::Ver4PatriciaTriePolicy>(
+                    backward::v402::Ver4PatriciaTriePolicy>(
                             dictFormatVersion, locale, attributeMap);
         }
-        case FormatUtils::VERSION_4:
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4_DEV: {
             return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
@@ -116,14 +115,13 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
         case FormatUtils::VERSION_2:
             AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
             break;
-        case FormatUtils::VERSION_401: {
+        case FormatUtils::VERSION_4: {
-            return newPolicyForV4Dict<backward::v401::Ver4DictConstants,
+            return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
-                    backward::v401::Ver4DictBuffers,
+                    backward::v402::Ver4DictBuffers,
-                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
+                    backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
-                    backward::v401::Ver4PatriciaTriePolicy>(
+                    backward::v402::Ver4PatriciaTriePolicy>(
                             headerFilePath, formatVersion, std::move(mmappedBuffer));
         }
-        case FormatUtils::VERSION_4:
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4_DEV: {
             return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
@@ -179,7 +177,6 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
         case FormatUtils::VERSION_2:
             return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
                     new PatriciaTriePolicy(std::move(mmappedBuffer)));
-        case FormatUtils::VERSION_401:
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4:
         case FormatUtils::VERSION_4_DEV:

native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp

@@ -23,7 +23,7 @@
 #include <sys/types.h>
 
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v402/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@@ -41,12 +41,11 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
     TimeKeeper::setCurrentTime();
     const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
     switch (formatVersion) {
-        case FormatUtils::VERSION_401:
-            return createEmptyV4DictFile<backward::v401::Ver4DictConstants,
-                    backward::v401::Ver4DictBuffers,
-                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr>(
-                            filePath, localeAsCodePointVector, attributeMap, formatVersion);
         case FormatUtils::VERSION_4:
+            return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
+                    backward::v402::Ver4DictBuffers,
+                    backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
+                            filePath, localeAsCodePointVector, attributeMap, formatVersion);
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4_DEV:
             return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,

native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp

@@ -29,8 +29,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
     switch (formatVersion) {
         case VERSION_2:
             return VERSION_2;
-        case VERSION_401:
-            return VERSION_401;
         case VERSION_4_ONLY_FOR_TESTING:
             return VERSION_4_ONLY_FOR_TESTING;
         case VERSION_4:
@@ -52,7 +50,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
     const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
     switch (magicNumber) {
         case MAGIC_NUMBER:
-            // Version 2 header is as follows:
+            // The layout of the header is as follows:
             // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
             // Dictionary format version number (2 bytes)
             // Options (2 bytes)
@@ -60,19 +58,7 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
             // Conceptually this converts the hardcoded value of the bytes in the file into
             // the symbolic value we use in the code. But we want the constants to be the
             // same so we use them for both here.
-            if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
+            return getFormatVersion(ByteArrayUtils::readUint16(dict, 4));
-                return VERSION_2;
-            } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_401) {
-                return VERSION_401;
-            } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_ONLY_FOR_TESTING) {
-                return VERSION_4_ONLY_FOR_TESTING;
-            } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
-                return VERSION_4;
-            } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) {
-                return VERSION_4_DEV;
-            } else {
-                return UNKNOWN_VERSION;
-            }
         default:
             return UNKNOWN_VERSION;
     }

native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h

@@ -32,7 +32,6 @@ class FormatUtils {
         // These MUST have the same values as the relevant constants in FormatSpec.java.
         VERSION_2 = 2,
         VERSION_4_ONLY_FOR_TESTING = 399,
-        VERSION_401 = 401,
         VERSION_4 = 402,
         VERSION_4_DEV = 403,
         UNKNOWN_VERSION = -1