From 26874845859d9d8ee229277a3fb49bf5172c9ee0 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Tue, 12 Nov 2013 15:36:02 +0900 Subject: [PATCH] Create empty ver4 dictionary buffer on memory. Bug: 11073222 Change-Id: Ia1f4d7782e019fdc6a4162f942f433d8b150b49d --- ...oid_inputmethod_latin_BinaryDictionary.cpp | 4 +- ...y_structure_with_buffer_policy_factory.cpp | 8 ++-- .../v4/content/bigram_dict_content.h | 6 ++- .../v4/content/probability_dict_content.h | 4 +- .../v4/content/shortcut_dict_content.h | 6 ++- .../v4/content/single_dict_content.h | 13 ++++-- .../v4/content/sparse_table_dict_content.h | 18 ++++++-- .../content/terminal_position_lookup_table.h | 4 +- .../structure/v4/ver4_dict_buffers.h | 43 ++++++++++++++----- .../structure/v4/ver4_dict_constants.cpp | 4 ++ .../structure/v4/ver4_dict_constants.h | 2 + .../v4/ver4_patricia_trie_policy.cpp | 22 +++++----- .../structure/v4/ver4_patricia_trie_policy.h | 15 +++---- .../utils/dict_file_writing_utils.cpp | 23 ++++++++-- .../utils/dict_file_writing_utils.h | 3 ++ 15 files changed, 125 insertions(+), 50 deletions(-) diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 3becc7e39..c4383d754 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -86,10 +86,10 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s char sourceDirChars[sourceDirUtf8Length + 1]; env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars); sourceDirChars[sourceDirUtf8Length] = '\0'; - DictionaryStructureWithBufferPolicy::StructurePoilcyPtr dictionaryStructureWithBufferPolicy( + DictionaryStructureWithBufferPolicy::StructurePoilcyPtr dictionaryStructureWithBufferPolicy = DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy( sourceDirChars, static_cast(dictOffset), static_cast(dictSize), - isUpdatable == JNI_TRUE)); + isUpdatable == JNI_TRUE); if (!dictionaryStructureWithBufferPolicy.get()) { return 0; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index 3ab6a8e21..063b84cbf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -35,8 +35,8 @@ namespace latinime { const int bufOffset, const int size, const bool isUpdatable) { // Allocated buffer in MmapedBuffer::newBuffer() will be freed in the destructor of // MmappedBufferWrapper if the instance has the responsibility. - MmappedBuffer::MmappedBufferPtr mmappedBuffer(MmappedBuffer::openBuffer(path, bufOffset, size, - isUpdatable)); + MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size, + isUpdatable); if (!mmappedBuffer.get()) { return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0); } @@ -58,8 +58,8 @@ namespace latinime { } // Removing extension to get the base path. dictDirPath.erase(pos); - const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( - Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer)); + const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = + Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer); if (!dictBuffers.get()->isValid()) { AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements."); ASSERT(false); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h index a112667e2..bc9e4b619 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h @@ -33,6 +33,10 @@ class BigramDictContent : public SparseTableDictContent { Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE) {} + BigramDictContent() + : SparseTableDictContent(Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, + Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE) {} + void getBigramEntryAndAdvancePosition(int *const outProbability, bool *const outHasNext, int *const outTargetTerminalId, int *const bigramEntryPos) const; @@ -56,7 +60,7 @@ class BigramDictContent : public SparseTableDictContent { bool copyBigramList(const int bigramListPos, const int toPos); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictContent); + DISALLOW_COPY_AND_ASSIGN(BigramDictContent); int createAndGetBigramFlags(const int probability, const bool hasNext) const { return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h index e85bbe18e..c109cbf51 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -31,6 +31,8 @@ class ProbabilityDictContent : public SingleDictContent { : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable) {} + ProbabilityDictContent() {} + int getProbability(const int terminalId) const { if (terminalId < 0 || terminalId >= getSize()) { return NOT_A_PROBABILITY; @@ -61,7 +63,7 @@ class ProbabilityDictContent : public SingleDictContent { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityDictContent); + DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent); int getSize() const { return getBuffer()->getTailPosition() / (Ver4DictConstants::PROBABILITY_SIZE diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h index c10fbcb2a..8463a1753 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h @@ -33,6 +33,10 @@ class ShortcutDictContent : public SparseTableDictContent { Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {} + ShortcutDictContent() + : SparseTableDictContent(Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, + Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {} + void getShortcutEntryAndAdvancePosition(const int maxCodePointCount, int *const outCodePoint, int *const outCodePointCount, int *const outShortcutFlags, int *const shortcutEntryPos) const { @@ -57,7 +61,7 @@ class ShortcutDictContent : public SparseTableDictContent { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(ShortcutDictContent); + DISALLOW_COPY_AND_ASSIGN(ShortcutDictContent); }; } // namespace latinime #endif /* LATINIME_SHORTCUT_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index 4cb96da6a..7669c1eca 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -19,6 +19,7 @@ #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" @@ -31,12 +32,17 @@ class SingleDictContent : public DictContent { : mMmappedBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)), mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0, mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0, - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {} + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + mIsValid(mMmappedBuffer.get() != 0) {} + + SingleDictContent() + : mMmappedBuffer(0), mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mIsValid(true) {} virtual ~SingleDictContent() {} virtual bool isValid() const { - return mMmappedBuffer.get() != 0; + return mIsValid; } protected: @@ -49,10 +55,11 @@ class SingleDictContent : public DictContent { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(SingleDictContent); + DISALLOW_COPY_AND_ASSIGN(SingleDictContent); const MmappedBuffer::MmappedBufferPtr mMmappedBuffer; BufferWithExtendableBuffer mExpandableContentBuffer; + const bool mIsValid; }; } // namespace latinime #endif /* LATINIME_SINGLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h index fe9efe874..5ae5f0ff1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h @@ -19,6 +19,7 @@ #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" #include "suggest/policyimpl/dictionary/utils/sparse_table.h" @@ -49,13 +50,22 @@ class SparseTableDictContent : public DictContent { mContentBuffer.get() ? mContentBuffer.get()->getBufferSize() : 0, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, - sparseTableBlockSize, sparseTableDataSize) {} + sparseTableBlockSize, sparseTableDataSize), + mIsValid(mLookupTableBuffer.get() != 0 && mAddressTableBuffer.get() != 0 + && mContentBuffer.get() != 0) {} + + SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize) + : mLookupTableBuffer(0), mAddressTableBuffer(0), mContentBuffer(0), + mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, + sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {} virtual ~SparseTableDictContent() {} virtual bool isValid() const { - return mLookupTableBuffer.get() != 0 && mAddressTableBuffer.get() != 0 - && mContentBuffer.get() != 0; + return mIsValid; } protected: @@ -78,7 +88,6 @@ class SparseTableDictContent : public DictContent { private: DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent); - // TODO: Have sparse table. const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer; const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer; const MmappedBuffer::MmappedBufferPtr mContentBuffer; @@ -86,6 +95,7 @@ class SparseTableDictContent : public DictContent { BufferWithExtendableBuffer mExpandableAddressTableBuffer; BufferWithExtendableBuffer mExpandableContentBuffer; SparseTable mAddressLookupTable; + const bool mIsValid; }; } // namespace latinime #endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index f6ced31b4..e016a2b5f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -38,6 +38,8 @@ class TerminalPositionLookupTable : public SingleDictContent { / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE), mHeaderRegionSize(headerRegionSize) {} + TerminalPositionLookupTable() : mSize(0), mHeaderRegionSize(0) {} + int getTerminalPtNodePosition(const int terminalId) const { if (terminalId < 0 || terminalId >= mSize) { return NOT_A_DICT_POS; @@ -66,7 +68,7 @@ class TerminalPositionLookupTable : public SingleDictContent { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalPositionLookupTable); + DISALLOW_COPY_AND_ASSIGN(TerminalPositionLookupTable); int mSize; const int mHeaderRegionSize; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index b9e56a254..e468be591 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -33,27 +33,30 @@ class Ver4DictBuffers { public: typedef ExclusiveOwnershipPointer Ver4DictBuffersPtr; - static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, + static AK_FORCE_INLINE Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &dictBuffer) { const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false; return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); } + static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() { + return Ver4DictBuffersPtr(new Ver4DictBuffers()); + } + AK_FORCE_INLINE bool isValid() const { return mDictBuffer.get() != 0 && mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid() && mBigramDictContent.isValid() && mShortcutDictContent.isValid(); } - AK_FORCE_INLINE uint8_t *getRawDictBuffer() const { - return mDictBuffer.get()->getBuffer(); + AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() { + return &mExpandableHeaderBuffer; } - AK_FORCE_INLINE int getRawDictBufferSize() const { - return mDictBuffer.get()->getBufferSize(); + AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableTrieBuffer() { + return &mExpandableTrieBuffer; } - AK_FORCE_INLINE TerminalPositionLookupTable *getUpdatableTerminalPositionLookupTable() { return &mTerminalPositionLookupTable; } @@ -86,21 +89,41 @@ class Ver4DictBuffers { return mIsUpdatable; } + bool flush(const char *const dictDirPath) { + // TODO: Implement. + return false; + } + private: - DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictBuffers); + DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) : mDictBuffer(dictBuffer), - // TODO: Quit using getHeaderSize. - mTerminalPositionLookupTable(dictDirPath, isUpdatable, - HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())), + mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())), + mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize, + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize, + dictBuffer.get()->getBufferSize() - mHeaderSize, + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + // TODO: Quit using header size. + mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize), mProbabilityDictContent(dictDirPath, isUpdatable), mBigramDictContent(dictDirPath, isUpdatable), mShortcutDictContent(dictDirPath, isUpdatable), mIsUpdatable(isUpdatable) {} + AK_FORCE_INLINE Ver4DictBuffers() + : mDictBuffer(0), mHeaderSize(0), + mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mTerminalPositionLookupTable(), mProbabilityDictContent(), + mBigramDictContent(), mShortcutDictContent(), mIsUpdatable(true) {} + const MmappedBuffer::MmappedBufferPtr mDictBuffer; + const int mHeaderSize; + BufferWithExtendableBuffer mExpandableHeaderBuffer; + BufferWithExtendableBuffer mExpandableTrieBuffer; TerminalPositionLookupTable mTerminalPositionLookupTable; ProbabilityDictContent mProbabilityDictContent; BigramDictContent mBigramDictContent; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index ced76d25d..af13a374a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -30,6 +30,10 @@ const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".sh const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION = ".shortcut_index_shortcut"; +// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets. +// TODO: Make MAX_DICTIONARY_SIZE 8MB. +const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024; + const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1; const int Ver4DictConstants::PROBABILITY_SIZE = 1; const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index b00821373..cfb7740be 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -34,6 +34,8 @@ class Ver4DictConstants { static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION; static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION; + static const int MAX_DICTIONARY_SIZE; + static const int NOT_A_TERMINAL_ID; static const int PROBABILITY_SIZE; static const int FLAGS_IN_PROBABILITY_FILE_SIZE; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 40cc3d00b..698483a79 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -28,14 +28,14 @@ namespace latinime { const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024; const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = - DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; + Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS; void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const dicNode, DicNodeVector *const childDicNodes) const { if (!dicNode->hasChildren()) { return; } - DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader); + DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); while (!readingHelper.isEnd()) { const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams(); @@ -63,7 +63,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) const { - DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader); + DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodePos(ptNodePos); return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount( maxCodePointCount, outCodePoints, outUnigramProbability); @@ -71,7 +71,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { - DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader); + DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); } @@ -135,12 +135,12 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); return false; } - if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d", - mDictBuffer.getTailPosition()); + mDictBuffer->getTailPosition()); return false; } - DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader); + DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); bool addedNewUnigram = false; if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, @@ -160,9 +160,9 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); return false; } - if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d", - mDictBuffer.getTailPosition()); + mDictBuffer->getTailPosition()); return false; } const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0, @@ -192,9 +192,9 @@ bool Ver4PatriciaTriePolicy::removeBigramWords(const int *const word0, const int AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); return false; } - if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d", - mDictBuffer.getTailPosition()); + mDictBuffer->getTailPosition()); return false; } const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 93cc4b7a2..e8fdf5513 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -38,18 +38,17 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers) : mBuffers(buffers), - mHeaderPolicy(mBuffers.get()->getRawDictBuffer(), FormatUtils::VERSION_4), - mDictBuffer(mBuffers.get()->getRawDictBuffer() + mHeaderPolicy.getSize(), - mBuffers.get()->getRawDictBufferSize() - mHeaderPolicy.getSize(), - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + mHeaderPolicy(mBuffers.get()->getWritableHeaderBuffer()->getBuffer( + false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4), + mDictBuffer(mBuffers.get()->getWritableTrieBuffer()), mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), mShortcutPolicy(mBuffers.get()->getShortcutDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), - mNodeReader(&mDictBuffer, mBuffers.get()->getProbabilityDictContent()), - mNodeWriter(&mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy, + mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()), + mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy, &mShortcutPolicy), - mUpdatingHelper(&mDictBuffer, &mNodeReader, &mNodeWriter, + mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter, mHeaderPolicy.isDecayingDict()), mUnigramCount(mHeaderPolicy.getUnigramCount()), mBigramCount(mHeaderPolicy.getBigramCount()) {}; @@ -115,7 +114,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { Ver4DictBuffers::Ver4DictBuffersPtr mBuffers; const HeaderPolicy mHeaderPolicy; - BufferWithExtendableBuffer mDictBuffer; + BufferWithExtendableBuffer *const mDictBuffer; Ver4BigramListPolicy mBigramPolicy; Ver4ShortcutListPolicy mShortcutPolicy; Ver4PatriciaTrieNodeReader mNodeReader; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index b48e5b005..40f7d1f5c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -21,6 +21,7 @@ #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" @@ -34,7 +35,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = case 3: return createEmptyV3DictFile(filePath, attributeMap); case 4: - // TODO: Support version 4 dictionary format. + return createEmptyV4DictFile(filePath, attributeMap); return false; default: // Only version 3 dictionary is supported for now. @@ -58,6 +59,20 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = return flushAllHeaderAndBodyToFile(filePath, &headerBuffer, &bodyBuffer); } +/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const filePath, + const HeaderReadWriteUtils::AttributeMap *const attributeMap) { + Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers(); + HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap); + headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), + true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, + 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); + if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( + dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) { + return false; + } + return dictBuffers.get()->flush(filePath); +} + /* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath, BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) { const int tmpFileNameBufSize = strlen(filePath) @@ -69,21 +84,21 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); FILE *const file = fopen(tmpFileName, "wb"); if (!file) { - AKLOGE("Dictionary file %s cannnot be opened.", tmpFileName); + AKLOGE("Dictionary file %s cannot be opened.", tmpFileName); ASSERT(false); return false; } // Write the dictionary header. if (!writeBufferToFile(file, dictHeader)) { remove(tmpFileName); - AKLOGE("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); + AKLOGE("Dictionary header cannot be written. size: %d", dictHeader->getTailPosition()); ASSERT(false); return false; } // Write the dictionary body. if (!writeBufferToFile(file, dictBody)) { remove(tmpFileName); - AKLOGE("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); + AKLOGE("Dictionary body cannot be written. size: %d", dictBody->getTailPosition()); ASSERT(false); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h index bd4ac66fd..3291f98c7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -43,6 +43,9 @@ class DictFileWritingUtils { static bool createEmptyV3DictFile(const char *const filePath, const HeaderReadWriteUtils::AttributeMap *const attributeMap); + static bool createEmptyV4DictFile(const char *const filePath, + const HeaderReadWriteUtils::AttributeMap *const attributeMap); + static bool writeBufferToFile(FILE *const file, const BufferWithExtendableBuffer *const buffer); };