From 7116ea98f4002b79b517a695e37e01a78c84e97e Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Wed, 14 May 2014 22:37:05 +0900 Subject: [PATCH] Add VERSION_4_DEV(402) in native code. Bug: 13406708 Change-Id: I96cfacf524d670a6a5637a96a63bcd47aaf09ca0 --- .../dictionary/header/header_policy.h | 2 + .../header/header_read_write_utils.cpp | 1 + ...y_structure_with_buffer_policy_factory.cpp | 117 ++++++++++++------ ...ary_structure_with_buffer_policy_factory.h | 16 ++- .../utils/dict_file_writing_utils.cpp | 19 ++- .../utils/dict_file_writing_utils.h | 6 + .../dictionary/utils/format_utils.cpp | 4 + .../dictionary/utils/format_utils.h | 1 + 8 files changed, 118 insertions(+), 48 deletions(-) diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index da24302c2..479d15164 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -143,6 +143,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return FormatUtils::VERSION_4_ONLY_FOR_TESTING; case FormatUtils::VERSION_4: return FormatUtils::VERSION_4; + case FormatUtils::VERSION_4_DEV: + return FormatUtils::VERSION_4_DEV; default: return FormatUtils::UNKNOWN_VERSION; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index 2a9028a9e..a8f8f284b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -100,6 +100,7 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap; return false; case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: + case FormatUtils::VERSION_4_DEV: return buffer->writeUintAndAdvancePosition(version /* data */, HEADER_DICTIONARY_VERSION_SIZE, writingPos); default: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index c4d18608c..59f1f29e9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -19,6 +19,9 @@ #include #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" @@ -42,7 +45,7 @@ namespace latinime { if (isUpdatable) { AKLOGE("One file dictionaries don't support updating. path: %s", path); ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; } return newPolicyForFileDict(path, bufOffset, size); } @@ -54,26 +57,43 @@ namespace latinime { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion); switch (dictFormatVersion) { - case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: { - HeaderPolicy headerPolicy(dictFormatVersion, locale, attributeMap); - Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = - Ver4DictBuffers::createVer4DictBuffers(&headerPolicy, - Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE); - if (!DynamicPtWritingUtils::writeEmptyDictionary( - dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) { - AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); - } - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( - new Ver4PatriciaTriePolicy(std::move(dictBuffers))); + return newPolicyForOnMemoryV4Dict( + dictFormatVersion, locale, attributeMap); + } + case FormatUtils::VERSION_4_ONLY_FOR_TESTING: + case FormatUtils::VERSION_4_DEV: { + return newPolicyForOnMemoryV4Dict( + dictFormatVersion, locale, attributeMap); } default: AKLOGE("DICT: dictionary format %d is not supported for on memory dictionary", formatVersion); break; } - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; +} + +template +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyForOnMemoryV4Dict( + const FormatUtils::FORMAT_VERSION formatVersion, + const std::vector &locale, + const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { + HeaderPolicy headerPolicy(formatVersion, locale, attributeMap); + DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy, + DictConstants::MAX_DICT_EXTENDED_REGION_SIZE); + if (!DynamicPtWritingUtils::writeEmptyDictionary( + dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) { + AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); + return nullptr; + } + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( + new StructurePolicy(std::move(dictBuffers))); } /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr @@ -84,10 +104,10 @@ namespace latinime { getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath); // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of // MmappedBufferPtr if the instance has the responsibility. - MmappedBuffer::MmappedBufferPtr mmappedBuffer( - MmappedBuffer::openBuffer(headerFilePath, isUpdatable)); + MmappedBuffer::MmappedBufferPtr mmappedBuffer = + MmappedBuffer::openBuffer(headerFilePath, isUpdatable); if (!mmappedBuffer) { - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; } const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion( mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize()); @@ -95,34 +115,50 @@ namespace latinime { case FormatUtils::VERSION_2: AKLOGE("Given path is a directory but the format is version 2. path: %s", path); break; - case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: { - const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */; - char dictPath[dictDirPathBufSize]; - if (!FileUtils::getFilePathWithoutSuffix(headerFilePath, - Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) { - AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path); - ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); - } - Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( - Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer), - formatVersion)); - if (!dictBuffers || !dictBuffers->isValid()) { - AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s", - path); - ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); - } - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( - new Ver4PatriciaTriePolicy(std::move(dictBuffers))); + return newPolicyForV4Dict( + headerFilePath, formatVersion, std::move(mmappedBuffer)); + } + case FormatUtils::VERSION_4_ONLY_FOR_TESTING: + case FormatUtils::VERSION_4_DEV: { + return newPolicyForV4Dict( + headerFilePath, formatVersion, std::move(mmappedBuffer)); } default: AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path); break; } ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; +} + +template +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyForV4Dict( + const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion, + MmappedBuffer::MmappedBufferPtr &&mmappedBuffer) { + const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */; + char dictPath[dictDirPathBufSize]; + if (!FileUtils::getFilePathWithoutSuffix(headerFilePath, + DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) { + AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path); + ASSERT(false); + return nullptr; + } + DictBuffersPtr dictBuffers = + DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer), formatVersion); + if (!dictBuffers || !dictBuffers->isValid()) { + AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s", + path); + ASSERT(false); + return nullptr; + } + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( + new StructurePolicy(std::move(dictBuffers))); } /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr @@ -133,7 +169,7 @@ namespace latinime { MmappedBuffer::MmappedBufferPtr mmappedBuffer( MmappedBuffer::openBuffer(path, bufOffset, size, false /* isUpdatable */)); if (!mmappedBuffer) { - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; } switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize())) { @@ -142,6 +178,7 @@ namespace latinime { new PatriciaTriePolicy(std::move(mmappedBuffer))); case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: + case FormatUtils::VERSION_4_DEV: AKLOGE("Given path is a file but the format is version 4. path: %s", path); break; default: @@ -149,7 +186,7 @@ namespace latinime { break; } ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + return nullptr; } /* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir( diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h index f71447e23..768454d8d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h @@ -22,6 +22,8 @@ #include "defines.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" +#include "suggest/policyimpl/dictionary/utils/format_utils.h" +#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { @@ -32,16 +34,26 @@ class DictionaryStructureWithBufferPolicyFactory { const int size, const bool isUpdatable); static DictionaryStructureWithBufferPolicy::StructurePolicyPtr - newPolicyForOnMemoryDict(const int formatVersion, - const std::vector &locale, + newPolicyForOnMemoryDict(const int formatVersion, const std::vector &locale, const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); + template + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyForOnMemoryV4Dict(const FormatUtils::FORMAT_VERSION formatVersion, + const std::vector &locale, + const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap); + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr newPolicyForDirectoryDict(const char *const path, const bool isUpdatable); + template + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr newPolicyForV4Dict( + const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion, + MmappedBuffer::MmappedBufferPtr &&mmappedBuffer); + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr newPolicyForFileDict(const char *const path, const int bufOffset, const int size); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index 80970c7f8..105363db5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -23,6 +23,7 @@ #include #include "suggest/policyimpl/dictionary/header/header_policy.h" +#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -40,10 +41,16 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = TimeKeeper::setCurrentTime(); const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion); switch (formatVersion) { - case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4: - return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap, - formatVersion); + return createEmptyV4DictFile( + filePath, localeAsCodePointVector, attributeMap, formatVersion); + case FormatUtils::VERSION_4_ONLY_FOR_TESTING: + case FormatUtils::VERSION_4_DEV: + return createEmptyV4DictFile( + filePath, localeAsCodePointVector, attributeMap, formatVersion); default: AKLOGE("Cannot create dictionary %s because format version %d is not supported.", filePath, dictVersion); @@ -51,14 +58,14 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = } } +template /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath, const std::vector localeAsCodePointVector, const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap, const FormatUtils::FORMAT_VERSION formatVersion) { HeaderPolicy headerPolicy(formatVersion, localeAsCodePointVector, attributeMap); - Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( - Ver4DictBuffers::createVer4DictBuffers(&headerPolicy, - Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE)); + DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy, + DictConstants::MAX_DICT_EXTENDED_REGION_SIZE); headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */, dictBuffers->getWritableHeaderBuffer()); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h index a822989db..5df5856d2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -45,6 +45,12 @@ class DictFileWritingUtils { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils); + static bool createEmptyV401DictFile(const char *const filePath, + const std::vector localeAsCodePointVector, + const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap, + const FormatUtils::FORMAT_VERSION formatVersion); + + template static bool createEmptyV4DictFile(const char *const filePath, const std::vector localeAsCodePointVector, const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp index a8518cdca..ba405b07e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp @@ -33,6 +33,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; return VERSION_4_ONLY_FOR_TESTING; case VERSION_4: return VERSION_4; + case VERSION_4_DEV: + return VERSION_4_DEV; default: return UNKNOWN_VERSION; } @@ -62,6 +64,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; return VERSION_4_ONLY_FOR_TESTING; } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { return VERSION_4; + } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) { + return VERSION_4_DEV; } else { return UNKNOWN_VERSION; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 20dfb9d8c..c47f30ca4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -33,6 +33,7 @@ class FormatUtils { VERSION_2 = 2, VERSION_4_ONLY_FOR_TESTING = 399, VERSION_4 = 401, + VERSION_4_DEV = 402, UNKNOWN_VERSION = -1 };