nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Add VERSION_4_DEV(402) in native code."

main
Keisuke Kuroyanagi 2014-05-14 13:40:10 +00:00 committed by Android (Google) Code Review
parent 28b896fe35 7116ea98f4
commit 91d7538225
8 changed files with 118 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -143,6 +143,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
case FormatUtils::VERSION_4:
return FormatUtils::VERSION_4;
case FormatUtils::VERSION_4_DEV:
return FormatUtils::VERSION_4_DEV;
default:
return FormatUtils::UNKNOWN_VERSION;
}

1
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -100,6 +100,7 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
return false;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:
return buffer->writeUintAndAdvancePosition(version /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
default:

117
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -19,6 +19,9 @@
#include <climits>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
@ -42,7 +45,7 @@ namespace latinime {
if (isUpdatable) {
AKLOGE("One file dictionaries don't support updating. path: %s", path);
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
return newPolicyForFileDict(path, bufOffset, size);
}
@ -54,26 +57,43 @@ namespace latinime {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
switch (dictFormatVersion) {
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: {
HeaderPolicy headerPolicy(dictFormatVersion, locale, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
if (!DynamicPtWritingUtils::writeEmptyDictionary(
dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) {
AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
}
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new Ver4PatriciaTriePolicy(std::move(dictBuffers)));
return newPolicyForOnMemoryV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
dictFormatVersion, locale, attributeMap);
}
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
dictFormatVersion, locale, attributeMap);
}
default:
AKLOGE("DICT: dictionary format %d is not supported for on memory dictionary",
formatVersion);
break;
}
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
DictionaryStructureWithBufferPolicyFactory::newPolicyForOnMemoryV4Dict(
const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<int> &locale,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
HeaderPolicy headerPolicy(formatVersion, locale, attributeMap);
DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy,
DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
if (!DynamicPtWritingUtils::writeEmptyDictionary(
dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) {
AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
return nullptr;
}
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new StructurePolicy(std::move(dictBuffers)));
}
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
@ -84,10 +104,10 @@ namespace latinime {
getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath);
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
// MmappedBufferPtr if the instance has the responsibility.
MmappedBuffer::MmappedBufferPtr mmappedBuffer(
MmappedBuffer::openBuffer(headerFilePath, isUpdatable));
MmappedBuffer::MmappedBufferPtr mmappedBuffer =
MmappedBuffer::openBuffer(headerFilePath, isUpdatable);
if (!mmappedBuffer) {
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
@ -95,34 +115,50 @@ namespace latinime {
case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
break;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: {
const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
char dictPath[dictDirPathBufSize];
if (!FileUtils::getFilePathWithoutSuffix(headerFilePath,
Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
}
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer),
formatVersion));
if (!dictBuffers || !dictBuffers->isValid()) {
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
path);
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
}
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new Ver4PatriciaTriePolicy(std::move(dictBuffers)));
return newPolicyForV4Dict<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr,
backward::v401::Ver4PatriciaTriePolicy>(
headerFilePath, formatVersion, std::move(mmappedBuffer));
}
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: {
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
headerFilePath, formatVersion, std::move(mmappedBuffer));
}
default:
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
break;
}
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
DictionaryStructureWithBufferPolicyFactory::newPolicyForV4Dict(
const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion,
MmappedBuffer::MmappedBufferPtr &&mmappedBuffer) {
const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
char dictPath[dictDirPathBufSize];
if (!FileUtils::getFilePathWithoutSuffix(headerFilePath,
DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
ASSERT(false);
return nullptr;
}
DictBuffersPtr dictBuffers =
DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer), formatVersion);
if (!dictBuffers || !dictBuffers->isValid()) {
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
path);
ASSERT(false);
return nullptr;
}
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new StructurePolicy(std::move(dictBuffers)));
}
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
@ -133,7 +169,7 @@ namespace latinime {
MmappedBuffer::MmappedBufferPtr mmappedBuffer(
MmappedBuffer::openBuffer(path, bufOffset, size, false /* isUpdatable */));
if (!mmappedBuffer) {
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
mmappedBuffer->getBufferSize())) {
@ -142,6 +178,7 @@ namespace latinime {
new PatriciaTriePolicy(std::move(mmappedBuffer)));
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
case FormatUtils::VERSION_4_DEV:
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
break;
default:
@ -149,7 +186,7 @@ namespace latinime {
break;
}
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
return nullptr;
}
/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir(

16
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h
View File

@ -22,6 +22,8 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
@ -32,16 +34,26 @@ class DictionaryStructureWithBufferPolicyFactory {
const int size, const bool isUpdatable);
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
newPolicyForOnMemoryDict(const int formatVersion,
const std::vector<int> &locale,
newPolicyForOnMemoryDict(const int formatVersion, const std::vector<int> &locale,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
newPolicyForOnMemoryV4Dict(const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<int> &locale,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
newPolicyForDirectoryDict(const char *const path, const bool isUpdatable);
template<class DictConstants, class DictBuffers, class DictBuffersPtr, class StructurePolicy>
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr newPolicyForV4Dict(
const char *const headerFilePath, const FormatUtils::FORMAT_VERSION formatVersion,
MmappedBuffer::MmappedBufferPtr &&mmappedBuffer);
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
newPolicyForFileDict(const char *const path, const int bufOffset, const int size);

19
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -23,6 +23,7 @@
#include <sys/types.h>
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/backward/v401/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
@ -40,10 +41,16 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
TimeKeeper::setCurrentTime();
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
switch (formatVersion) {
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4:
return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap,
formatVersion);
return createEmptyV4DictFile<backward::v401::Ver4DictConstants,
backward::v401::Ver4DictBuffers,
backward::v401::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV:
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion);
default:
AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
filePath, dictVersion);
@ -51,14 +58,14 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
}
}
template<class DictConstants, class DictBuffers, class DictBuffersPtr>
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
const FormatUtils::FORMAT_VERSION formatVersion) {
HeaderPolicy headerPolicy(formatVersion, localeAsCodePointVector, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE));
DictBuffersPtr dictBuffers = DictBuffers::createVer4DictBuffers(&headerPolicy,
DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */,
0 /* extendedRegionSize */, dictBuffers->getWritableHeaderBuffer());

6
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View File

@ -45,6 +45,12 @@ class DictFileWritingUtils {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
static bool createEmptyV401DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
const FormatUtils::FORMAT_VERSION formatVersion);
template<class DictConstants, class DictBuffers, class DictBuffersPtr>
static bool createEmptyV4DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,

4
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp
View File

@ -33,6 +33,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
return VERSION_4_ONLY_FOR_TESTING;
case VERSION_4:
return VERSION_4;
case VERSION_4_DEV:
return VERSION_4_DEV;
default:
return UNKNOWN_VERSION;
}
@ -62,6 +64,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
return VERSION_4_ONLY_FOR_TESTING;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_DEV) {
return VERSION_4_DEV;
} else {
return UNKNOWN_VERSION;
}

1
native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h
View File

@ -33,6 +33,7 @@ class FormatUtils {
VERSION_2 = 2,
VERSION_4_ONLY_FOR_TESTING = 399,
VERSION_4 = 401,
VERSION_4_DEV = 402,
UNKNOWN_VERSION = -1
};