nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Read hasHistoricalInfo from dictionary header."

main
Keisuke Kuroyanagi 2013-12-04 07:30:37 +00:00 committed by Android (Google) Code Review
parent cddb753b3e f7bcb762d1
commit b5ebb29f69
4 changed files with 29 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -68,6 +68,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {} &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
// Temporary dummy header.
HeaderPolicy()
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
~HeaderPolicy() {} ~HeaderPolicy() {}
virtual int getFormatVersionNumber() const { virtual int getFormatVersionNumber() const {
@ -139,7 +146,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int unigramCount, const int bigramCount, const int extendedRegionSize) const; const int unigramCount, const int bigramCount, const int extendedRegionSize) const;
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy); DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
static const char *const IS_DECAYING_DICT_KEY; static const char *const IS_DECAYING_DICT_KEY;

32
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View File

@ -18,7 +18,7 @@
#define LATINIME_VER4_DICT_BUFFER_H #define LATINIME_VER4_DICT_BUFFER_H
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" #include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
@ -39,8 +39,9 @@ class Ver4DictBuffers {
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
} }
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() { static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
return Ver4DictBuffersPtr(new Ver4DictBuffers()); const HeaderPolicy *const headerPolicy) {
return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy));
} }
AK_FORCE_INLINE bool isValid() const { AK_FORCE_INLINE bool isValid() const {
@ -118,30 +119,31 @@ class Ver4DictBuffers {
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer), : mDictBuffer(dictBuffer),
mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())), mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize, mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize, mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(),
dictBuffer.get()->getBufferSize() - mHeaderSize, dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
// TODO: Quit using header size. // TODO: Quit using header size.
mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize), mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()),
mProbabilityDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable), mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(),
mBigramDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable), isUpdatable),
mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), isUpdatable),
mShortcutDictContent(dictDirPath, isUpdatable), mShortcutDictContent(dictDirPath, isUpdatable),
mIsUpdatable(isUpdatable) {} mIsUpdatable(isUpdatable) {}
AK_FORCE_INLINE Ver4DictBuffers() AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
: mDictBuffer(0), mHeaderSize(0), : mDictBuffer(0), mHeaderPolicy(),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mTerminalPositionLookupTable(), mTerminalPositionLookupTable(),
mProbabilityDictContent(false /* hasHistoricalInfo */), mProbabilityDictContent(headerPolicy->hasHistricalInfoOfWords()),
mBigramDictContent(false /* hasHistoricalInfo */), mShortcutDictContent(), mBigramDictContent(headerPolicy->hasHistricalInfoOfWords()), mShortcutDictContent(),
mIsUpdatable(true) {} mIsUpdatable(true) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer; const MmappedBuffer::MmappedBufferPtr mDictBuffer;
const int mHeaderSize; const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mExpandableHeaderBuffer; BufferWithExtendableBuffer mExpandableHeaderBuffer;
BufferWithExtendableBuffer mExpandableTrieBuffer; BufferWithExtendableBuffer mExpandableTrieBuffer;
TerminalPositionLookupTable mTerminalPositionLookupTable; TerminalPositionLookupTable mTerminalPositionLookupTable;

3
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp
View File

@ -55,7 +55,8 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
const char *const trieFilePath, const HeaderPolicy *const headerPolicy, const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
const bool needsToDecay) { const bool needsToDecay) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers()); Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
int unigramCount = 0; int unigramCount = 0;
int bigramCount = 0; int bigramCount = 0;
if (needsToDecay) { if (needsToDecay) {

3
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -43,8 +43,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath, /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) { const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap); HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);