Merge "Read hasHistoricalInfo from dictionary header."
commit
b5ebb29f69
|
@ -68,6 +68,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
|
|||
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
|
||||
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
|
||||
|
||||
// Temporary dummy header.
|
||||
HeaderPolicy()
|
||||
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
|
||||
mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
|
||||
mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
|
||||
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
|
||||
|
||||
~HeaderPolicy() {}
|
||||
|
||||
virtual int getFormatVersionNumber() const {
|
||||
|
@ -139,7 +146,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
|
|||
const int unigramCount, const int bigramCount, const int extendedRegionSize) const;
|
||||
|
||||
private:
|
||||
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);
|
||||
DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
|
||||
|
||||
static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
|
||||
static const char *const IS_DECAYING_DICT_KEY;
|
||||
|
|
|
@ -18,7 +18,7 @@
|
|||
#define LATINIME_VER4_DICT_BUFFER_H
|
||||
|
||||
#include "defines.h"
|
||||
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
|
||||
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
||||
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
|
||||
#include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h"
|
||||
#include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
|
||||
|
@ -39,8 +39,9 @@ class Ver4DictBuffers {
|
|||
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
|
||||
}
|
||||
|
||||
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() {
|
||||
return Ver4DictBuffersPtr(new Ver4DictBuffers());
|
||||
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
|
||||
const HeaderPolicy *const headerPolicy) {
|
||||
return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy));
|
||||
}
|
||||
|
||||
AK_FORCE_INLINE bool isValid() const {
|
||||
|
@ -118,30 +119,31 @@ class Ver4DictBuffers {
|
|||
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
|
||||
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
|
||||
: mDictBuffer(dictBuffer),
|
||||
mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())),
|
||||
mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize,
|
||||
mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
|
||||
mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
|
||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||
mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize,
|
||||
dictBuffer.get()->getBufferSize() - mHeaderSize,
|
||||
mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(),
|
||||
dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(),
|
||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||
// TODO: Quit using header size.
|
||||
mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize),
|
||||
mProbabilityDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
|
||||
mBigramDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable),
|
||||
mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()),
|
||||
mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(),
|
||||
isUpdatable),
|
||||
mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), isUpdatable),
|
||||
mShortcutDictContent(dictDirPath, isUpdatable),
|
||||
mIsUpdatable(isUpdatable) {}
|
||||
|
||||
AK_FORCE_INLINE Ver4DictBuffers()
|
||||
: mDictBuffer(0), mHeaderSize(0),
|
||||
AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
|
||||
: mDictBuffer(0), mHeaderPolicy(),
|
||||
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||
mTerminalPositionLookupTable(),
|
||||
mProbabilityDictContent(false /* hasHistoricalInfo */),
|
||||
mBigramDictContent(false /* hasHistoricalInfo */), mShortcutDictContent(),
|
||||
mProbabilityDictContent(headerPolicy->hasHistricalInfoOfWords()),
|
||||
mBigramDictContent(headerPolicy->hasHistricalInfoOfWords()), mShortcutDictContent(),
|
||||
mIsUpdatable(true) {}
|
||||
|
||||
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
|
||||
const int mHeaderSize;
|
||||
const HeaderPolicy mHeaderPolicy;
|
||||
BufferWithExtendableBuffer mExpandableHeaderBuffer;
|
||||
BufferWithExtendableBuffer mExpandableTrieBuffer;
|
||||
TerminalPositionLookupTable mTerminalPositionLookupTable;
|
||||
|
|
|
@ -55,7 +55,8 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
|
|||
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
|
||||
const char *const trieFilePath, const HeaderPolicy *const headerPolicy,
|
||||
const bool needsToDecay) {
|
||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers());
|
||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
|
||||
Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
|
||||
int unigramCount = 0;
|
||||
int bigramCount = 0;
|
||||
if (needsToDecay) {
|
||||
|
|
|
@ -43,8 +43,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
|
|||
|
||||
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
|
||||
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
|
||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
|
||||
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
|
||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
|
||||
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
|
||||
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
|
||||
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
|
||||
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
|
||||
|
|
Loading…
Reference in New Issue