Merge "Add VERSION_4_DEV(402) in native code."

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

@@ -143,6 +143,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
                 return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
             case FormatUtils::VERSION_4:
                 return FormatUtils::VERSION_4;
+            case FormatUtils::VERSION_4_DEV:
+                return FormatUtils::VERSION_4_DEV;
             default:
                 return FormatUtils::UNKNOWN_VERSION;
         }

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp

@@ -100,6 +100,7 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
             return false;
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4:
+        case FormatUtils::VERSION_4_DEV:
             return buffer->writeUintAndAdvancePosition(version /* data */,
                     HEADER_DICTIONARY_VERSION_SIZE, writingPos);
         default:

native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp

@@ -19,6 +19,9 @@
 #include <climits>
 
 #include "defines.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
@@ -42,7 +45,7 @@ namespace latinime {
         if (isUpdatable) {
             AKLOGE("One file dictionaries don't support updating. path: %s", path);
             ASSERT(false);
-            return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+            return nullptr;
         }
         return newPolicyForFileDict(path, bufOffset, size);
     }
@@ -54,26 +57,43 @@ namespace latinime {
                 const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
     FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
     switch (dictFormatVersion) {
-        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4: {
-            HeaderPolicy headerPolicy(dictFormatVersion, locale, attributeMap);
+            return newPolicyForOnMemoryV4Dict<backward::v401::Ver4DictConstants,
-            Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
+                    backward::v401::Ver4DictBuffers,
-                    Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
+                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
-                            Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
+                    backward::v401::Ver4PatriciaTriePolicy>(
-            if (!DynamicPtWritingUtils::writeEmptyDictionary(
+                            dictFormatVersion, locale, attributeMap);
-                    dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) {
+        }
-                AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
+        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
-                return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+        case FormatUtils::VERSION_4_DEV: {
-            }
+            return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
-            return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
+                    Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
-                    new Ver4PatriciaTriePolicy(std::move(dictBuffers)));
+                            dictFormatVersion, locale, attributeMap);
         }
         default:
             AKLOGE("DICT: dictionary format %d is not supported for on memory dictionary",
                     formatVersion);
             break;
     }
-    return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+    return nullptr;
+}
+
+template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
+/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
+        DictionaryStructureWithBufferPolicyFactory::newPolicyForOnMemoryV4Dict(
+                const FormatUtils::FORMAT_VERSION formatVersion,
+                const std::vector<int> &locale,
+                const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
+    HeaderPolicy headerPolicy(formatVersion, locale, attributeMap);
+    DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy,
+            DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
+    if (!DynamicPtWritingUtils::writeEmptyDictionary(
+            dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) {
+        AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
+        return nullptr;
+    }
+    return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
+            new StructurePolicy(std::move(dictBuffers)));
 }
 
 /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
@@ -84,10 +104,10 @@ namespace latinime {
     getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath);
     // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
     // MmappedBufferPtr if the instance has the responsibility.
-    MmappedBuffer::MmappedBufferPtr mmappedBuffer(
+    MmappedBuffer::MmappedBufferPtr mmappedBuffer =
-            MmappedBuffer::openBuffer(headerFilePath, isUpdatable));
+            MmappedBuffer::openBuffer(headerFilePath, isUpdatable);
     if (!mmappedBuffer) {
-        return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+        return nullptr;
     }
     const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
             mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
@@ -95,34 +115,50 @@ namespace latinime {
         case FormatUtils::VERSION_2:
             AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
             break;
-        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4: {
-            const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
+            return newPolicyForV4Dict<backward::v401::Ver4DictConstants,
-            char dictPath[dictDirPathBufSize];
+                    backward::v401::Ver4DictBuffers,
-            if (!FileUtils::getFilePathWithoutSuffix(headerFilePath,
+                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
-                    Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
+                    backward::v401::Ver4PatriciaTriePolicy>(
-                AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
+                            headerFilePath, formatVersion, std::move(mmappedBuffer));
-                ASSERT(false);
+        }
-                return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
-            }
+        case FormatUtils::VERSION_4_DEV: {
-            Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
+            return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
-                    Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer),
+                    Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
-                            formatVersion));
+                            headerFilePath, formatVersion, std::move(mmappedBuffer));
-            if (!dictBuffers || !dictBuffers->isValid()) {
-                AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
-                        path);
-                ASSERT(false);
-                return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
-            }
-            return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
-                    new Ver4PatriciaTriePolicy(std::move(dictBuffers)));
         }
         default:
             AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
             break;
     }
     ASSERT(false);
-    return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+    return nullptr;
+}
+
+template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
+/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
+        DictionaryStructureWithBufferPolicyFactory::newPolicyForV4Dict(
+                const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion,
+                MmappedBuffer::MmappedBufferPtr &&mmappedBuffer) {
+    const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
+    char dictPath[dictDirPathBufSize];
+    if (!FileUtils::getFilePathWithoutSuffix(headerFilePath,
+            DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
+        AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
+        ASSERT(false);
+        return nullptr;
+    }
+    DictBuffersPtr dictBuffers =
+            DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer), formatVersion);
+    if (!dictBuffers || !dictBuffers->isValid()) {
+        AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
+                path);
+        ASSERT(false);
+        return nullptr;
+    }
+    return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
+            new StructurePolicy(std::move(dictBuffers)));
 }
 
 /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
@@ -133,7 +169,7 @@ namespace latinime {
     MmappedBuffer::MmappedBufferPtr mmappedBuffer(
             MmappedBuffer::openBuffer(path, bufOffset, size, false /* isUpdatable */));
     if (!mmappedBuffer) {
-        return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+        return nullptr;
     }
     switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
             mmappedBuffer->getBufferSize())) {
@@ -142,6 +178,7 @@ namespace latinime {
                     new PatriciaTriePolicy(std::move(mmappedBuffer)));
         case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4:
+        case FormatUtils::VERSION_4_DEV:
             AKLOGE("Given path is a file but the format is version 4. path: %s", path);
             break;
         default:
@@ -149,7 +186,7 @@ namespace latinime {
             break;
     }
     ASSERT(false);
-    return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
+    return nullptr;
 }
 
 /* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir(

native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h

@@ -22,6 +22,8 @@
 #include "defines.h"
 #include "suggest/core/policy/dictionary_header_structure_policy.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
+#include "suggest/policyimpl/dictionary/utils/format_utils.h"
+#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
 
 namespace latinime {
 
@@ -32,16 +34,26 @@ class DictionaryStructureWithBufferPolicyFactory {
                     const int size, const bool isUpdatable);
 
     static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
-            newPolicyForOnMemoryDict(const int formatVersion,
+            newPolicyForOnMemoryDict(const int formatVersion, const std::vector<int> &locale,
-                    const std::vector<int> &locale,
                     const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
 
+    template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
+    static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
+            newPolicyForOnMemoryV4Dict(const FormatUtils::FORMAT_VERSION formatVersion,
+                    const std::vector<int> &locale,
+                    const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
+
     static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
             newPolicyForDirectoryDict(const char *const path, const bool isUpdatable);
 
+    template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
+    static DictionaryStructureWithBufferPolicy::StructurePolicyPtr newPolicyForV4Dict(
+            const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion,
+                    MmappedBuffer::MmappedBufferPtr &&mmappedBuffer);
+
     static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
             newPolicyForFileDict(const char *const path, const int bufOffset, const int size);
 

native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp

@@ -23,6 +23,7 @@
 #include <sys/types.h>
 
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
+#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@@ -40,10 +41,16 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
     TimeKeeper::setCurrentTime();
     const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
     switch (formatVersion) {
-        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
         case FormatUtils::VERSION_4:
-            return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap,
+            return createEmptyV4DictFile<backward::v401::Ver4DictConstants,
-                    formatVersion);
+                    backward::v401::Ver4DictBuffers,
+                    backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr>(
+                            filePath, localeAsCodePointVector, attributeMap, formatVersion);
+        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
+        case FormatUtils::VERSION_4_DEV:
+            return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,
+                    Ver4DictBuffers::Ver4DictBuffersPtr>(
+                            filePath, localeAsCodePointVector, attributeMap, formatVersion);
         default:
             AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
                     filePath, dictVersion);
@@ -51,14 +58,14 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
     }
 }
 
+template<class DictConstants, class DictBuffers, class DictBuffersPtr>
 /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
         const std::vector<int> localeAsCodePointVector,
         const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
         const FormatUtils::FORMAT_VERSION formatVersion) {
     HeaderPolicy headerPolicy(formatVersion, localeAsCodePointVector, attributeMap);
-    Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
+    DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy,
-            Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
+            DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
-                    Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE));
     headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,
             0 /* unigramCount */, 0 /* bigramCount */,
             0 /* extendedRegionSize */, dictBuffers->getWritableHeaderBuffer());

native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h

@@ -45,6 +45,12 @@ class DictFileWritingUtils {
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
 
+    static bool createEmptyV401DictFile(const char *const filePath,
+            const std::vector<int> localeAsCodePointVector,
+            const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
+            const FormatUtils::FORMAT_VERSION formatVersion);
+
+    template<class DictConstants, class DictBuffers, class DictBuffersPtr>
     static bool createEmptyV4DictFile(const char *const filePath,
             const std::vector<int> localeAsCodePointVector,
             const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,

native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp

@@ -33,6 +33,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
             return VERSION_4_ONLY_FOR_TESTING;
         case VERSION_4:
             return VERSION_4;
+        case VERSION_4_DEV:
+            return VERSION_4_DEV;
         default:
             return UNKNOWN_VERSION;
     }
@@ -62,6 +64,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
                 return VERSION_4_ONLY_FOR_TESTING;
             } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
                 return VERSION_4;
+            } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) {
+                return VERSION_4_DEV;
             } else {
                 return UNKNOWN_VERSION;
             }

native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h

@@ -33,6 +33,7 @@ class FormatUtils {
         VERSION_2 = 2,
         VERSION_4_ONLY_FOR_TESTING = 399,
         VERSION_4 = 401,
+        VERSION_4_DEV = 402,
         UNKNOWN_VERSION = -1
     };