nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Create empty ver4 dictionary buffer on memory."

main
Keisuke Kuroyanagi 2013-11-13 08:19:13 +00:00 committed by Android (Google) Code Review
parent cc356d04e7 2687484585
commit c3a4c075c9
15 changed files with 125 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -86,10 +86,10 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
char sourceDirChars[sourceDirUtf8Length + 1];
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
sourceDirChars[sourceDirUtf8Length] = '\0';
DictionaryStructureWithBufferPolicy::StructurePoilcyPtr dictionaryStructureWithBufferPolicy(
DictionaryStructureWithBufferPolicy::StructurePoilcyPtr dictionaryStructureWithBufferPolicy =
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
isUpdatable == JNI_TRUE));
isUpdatable == JNI_TRUE);
if (!dictionaryStructureWithBufferPolicy.get()) {
return 0;
}

8
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -35,8 +35,8 @@ namespace latinime {
const int bufOffset, const int size, const bool isUpdatable) {
// Allocated buffer in MmapedBuffer::newBuffer() will be freed in the destructor of
// MmappedBufferWrapper if the instance has the responsibility.
MmappedBuffer::MmappedBufferPtr mmappedBuffer(MmappedBuffer::openBuffer(path, bufOffset, size,
isUpdatable));
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size,
isUpdatable);
if (!mmappedBuffer.get()) {
return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0);
}
@ -58,8 +58,8 @@ namespace latinime {
}
// Removing extension to get the base path.
dictDirPath.erase(pos);
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer));
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer);
if (!dictBuffers.get()->isValid()) {
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements.");
ASSERT(false);

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
View File

@ -33,6 +33,10 @@ class BigramDictContent : public SparseTableDictContent {
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE) {}
BigramDictContent()
: SparseTableDictContent(Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE) {}
void getBigramEntryAndAdvancePosition(int *const outProbability, bool *const outHasNext,
int *const outTargetTerminalId, int *const bigramEntryPos) const;
@ -56,7 +60,7 @@ class BigramDictContent : public SparseTableDictContent {
bool copyBigramList(const int bigramListPos, const int toPos);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictContent);
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
int createAndGetBigramFlags(const int probability, const bool hasNext) const {
return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
View File

@ -31,6 +31,8 @@ class ProbabilityDictContent : public SingleDictContent {
: SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION,
isUpdatable) {}
ProbabilityDictContent() {}
int getProbability(const int terminalId) const {
if (terminalId < 0 || terminalId >= getSize()) {
return NOT_A_PROBABILITY;
@ -61,7 +63,7 @@ class ProbabilityDictContent : public SingleDictContent {
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityDictContent);
DISALLOW_COPY_AND_ASSIGN(ProbabilityDictContent);
int getSize() const {
return getBuffer()->getTailPosition() / (Ver4DictConstants::PROBABILITY_SIZE

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
View File

@ -33,6 +33,10 @@ class ShortcutDictContent : public SparseTableDictContent {
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
ShortcutDictContent()
: SparseTableDictContent(Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
void getShortcutEntryAndAdvancePosition(const int maxCodePointCount,
int *const outCodePoint, int *const outCodePointCount, int *const outShortcutFlags,
int *const shortcutEntryPos) const {
@ -57,7 +61,7 @@ class ShortcutDictContent : public SparseTableDictContent {
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ShortcutDictContent);
DISALLOW_COPY_AND_ASSIGN(ShortcutDictContent);
};
} // namespace latinime
#endif /* LATINIME_SHORTCUT_DICT_CONTENT_H */

13
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View File

@ -19,6 +19,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
@ -31,12 +32,17 @@ class SingleDictContent : public DictContent {
: mMmappedBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)),
mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0,
mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mIsValid(mMmappedBuffer.get() != 0) {}
SingleDictContent()
: mMmappedBuffer(0), mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mIsValid(true) {}
virtual ~SingleDictContent() {}
virtual bool isValid() const {
return mMmappedBuffer.get() != 0;
return mIsValid;
}
protected:
@ -49,10 +55,11 @@ class SingleDictContent : public DictContent {
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SingleDictContent);
DISALLOW_COPY_AND_ASSIGN(SingleDictContent);
const MmappedBuffer::MmappedBufferPtr mMmappedBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SINGLE_DICT_CONTENT_H */

18
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View File

@ -19,6 +19,7 @@
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
@ -49,13 +50,22 @@ class SparseTableDictContent : public DictContent {
mContentBuffer.get() ? mContentBuffer.get()->getBufferSize() : 0,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize) {}
sparseTableBlockSize, sparseTableDataSize),
mIsValid(mLookupTableBuffer.get() != 0 && mAddressTableBuffer.get() != 0
&& mContentBuffer.get() != 0) {}
SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(0), mAddressTableBuffer(0), mContentBuffer(0),
mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {}
virtual ~SparseTableDictContent() {}
virtual bool isValid() const {
return mLookupTableBuffer.get() != 0 && mAddressTableBuffer.get() != 0
&& mContentBuffer.get() != 0;
return mIsValid;
}
protected:
@ -78,7 +88,6 @@ class SparseTableDictContent : public DictContent {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent);
// TODO: Have sparse table.
const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer;
const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer;
const MmappedBuffer::MmappedBufferPtr mContentBuffer;
@ -86,6 +95,7 @@ class SparseTableDictContent : public DictContent {
BufferWithExtendableBuffer mExpandableAddressTableBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
SparseTable mAddressLookupTable;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
View File

@ -38,6 +38,8 @@ class TerminalPositionLookupTable : public SingleDictContent {
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE),
mHeaderRegionSize(headerRegionSize) {}
TerminalPositionLookupTable() : mSize(0), mHeaderRegionSize(0) {}
int getTerminalPtNodePosition(const int terminalId) const {
if (terminalId < 0 || terminalId >= mSize) {
return NOT_A_DICT_POS;
@ -66,7 +68,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalPositionLookupTable);
DISALLOW_COPY_AND_ASSIGN(TerminalPositionLookupTable);
int mSize;
const int mHeaderRegionSize;

43
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View File

@ -33,27 +33,30 @@ class Ver4DictBuffers {
public:
typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr;
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
static AK_FORCE_INLINE Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer) {
const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false;
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
}
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() {
return Ver4DictBuffersPtr(new Ver4DictBuffers());
}
AK_FORCE_INLINE bool isValid() const {
return mDictBuffer.get() != 0 && mProbabilityDictContent.isValid()
&& mTerminalPositionLookupTable.isValid() && mBigramDictContent.isValid()
&& mShortcutDictContent.isValid();
}
AK_FORCE_INLINE uint8_t *getRawDictBuffer() const {
return mDictBuffer.get()->getBuffer();
AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() {
return &mExpandableHeaderBuffer;
}
AK_FORCE_INLINE int getRawDictBufferSize() const {
return mDictBuffer.get()->getBufferSize();
AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableTrieBuffer() {
return &mExpandableTrieBuffer;
}
AK_FORCE_INLINE TerminalPositionLookupTable *getUpdatableTerminalPositionLookupTable() {
return &mTerminalPositionLookupTable;
}
@ -86,21 +89,41 @@ class Ver4DictBuffers {
return mIsUpdatable;
}
bool flush(const char *const dictDirPath) {
// TODO: Implement.
return false;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictBuffers);
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer),
// TODO: Quit using getHeaderSize.
mTerminalPositionLookupTable(dictDirPath, isUpdatable,
HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())),
mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())),
mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize,
dictBuffer.get()->getBufferSize() - mHeaderSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
// TODO: Quit using header size.
mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize),
mProbabilityDictContent(dictDirPath, isUpdatable),
mBigramDictContent(dictDirPath, isUpdatable),
mShortcutDictContent(dictDirPath, isUpdatable),
mIsUpdatable(isUpdatable) {}
AK_FORCE_INLINE Ver4DictBuffers()
: mDictBuffer(0), mHeaderSize(0),
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mTerminalPositionLookupTable(), mProbabilityDictContent(),
mBigramDictContent(), mShortcutDictContent(), mIsUpdatable(true) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
const int mHeaderSize;
BufferWithExtendableBuffer mExpandableHeaderBuffer;
BufferWithExtendableBuffer mExpandableTrieBuffer;
TerminalPositionLookupTable mTerminalPositionLookupTable;
ProbabilityDictContent mProbabilityDictContent;
BigramDictContent mBigramDictContent;

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View File

@ -30,6 +30,10 @@ const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".sh
const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
".shortcut_index_shortcut";
// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
// TODO: Make MAX_DICTIONARY_SIZE 8MB.
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;
const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1;
const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -34,6 +34,8 @@ class Ver4DictConstants {
static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
static const int MAX_DICTIONARY_SIZE;
static const int NOT_A_TERMINAL_ID;
static const int PROBABILITY_SIZE;
static const int FLAGS_IN_PROBABILITY_FILE_SIZE;

22
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View File

@ -28,14 +28,14 @@ namespace latinime {
const int Ver4PatriciaTriePolicy::MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS = 1024;
const int Ver4PatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS =
DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
Ver4DictConstants::MAX_DICTIONARY_SIZE - MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const dicNode,
DicNodeVector *const childDicNodes) const {
if (!dicNode->hasChildren()) {
return;
}
DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader);
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos());
while (!readingHelper.isEnd()) {
const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams();
@ -63,7 +63,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d
int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
const int ptNodePos, const int maxCodePointCount, int *const outCodePoints,
int *const outUnigramProbability) const {
DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader);
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodePos(ptNodePos);
return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount(
maxCodePointCount, outCodePoints, outUnigramProbability);
@ -71,7 +71,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount(
int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord,
const int length, const bool forceLowerCaseSearch) const {
DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader);
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch);
}
@ -135,12 +135,12 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
return false;
}
if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d",
mDictBuffer.getTailPosition());
mDictBuffer->getTailPosition());
return false;
}
DynamicPatriciaTrieReadingHelper readingHelper(&mDictBuffer, &mNodeReader);
DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader);
readingHelper.initWithPtNodeArrayPos(getRootPosition());
bool addedNewUnigram = false;
if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability,
@ -160,9 +160,9 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false;
}
if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d",
mDictBuffer.getTailPosition());
mDictBuffer->getTailPosition());
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,
@ -192,9 +192,9 @@ bool Ver4PatriciaTriePolicy::removeBigramWords(const int *const word0, const int
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
return false;
}
if (mDictBuffer.getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
if (mDictBuffer->getTailPosition() >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
AKLOGE("The dictionary is too large to dynamically update. Dictionary size: %d",
mDictBuffer.getTailPosition());
mDictBuffer->getTailPosition());
return false;
}
const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0,

15
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View File

@ -38,18 +38,17 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers)
: mBuffers(buffers),
mHeaderPolicy(mBuffers.get()->getRawDictBuffer(), FormatUtils::VERSION_4),
mDictBuffer(mBuffers.get()->getRawDictBuffer() + mHeaderPolicy.getSize(),
mBuffers.get()->getRawDictBufferSize() - mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mHeaderPolicy(mBuffers.get()->getWritableHeaderBuffer()->getBuffer(
false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4),
mDictBuffer(mBuffers.get()->getWritableTrieBuffer()),
mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(),
mBuffers.get()->getTerminalPositionLookupTable()),
mShortcutPolicy(mBuffers.get()->getShortcutDictContent(),
mBuffers.get()->getTerminalPositionLookupTable()),
mNodeReader(&mDictBuffer, mBuffers.get()->getProbabilityDictContent()),
mNodeWriter(&mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy,
mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()),
mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy,
&mShortcutPolicy),
mUpdatingHelper(&mDictBuffer, &mNodeReader, &mNodeWriter,
mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter,
mHeaderPolicy.isDecayingDict()),
mUnigramCount(mHeaderPolicy.getUnigramCount()),
mBigramCount(mHeaderPolicy.getBigramCount()) {};
@ -115,7 +114,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
const HeaderPolicy mHeaderPolicy;
BufferWithExtendableBuffer mDictBuffer;
BufferWithExtendableBuffer *const mDictBuffer;
Ver4BigramListPolicy mBigramPolicy;
Ver4ShortcutListPolicy mShortcutPolicy;
Ver4PatriciaTrieNodeReader mNodeReader;

23
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -21,6 +21,7 @@
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
@ -34,7 +35,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
case 3:
return createEmptyV3DictFile(filePath, attributeMap);
case 4:
// TODO: Support version 4 dictionary format.
return createEmptyV4DictFile(filePath, attributeMap);
return false;
default:
// Only version 3 dictionary is supported for now.
@ -58,6 +59,20 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
return flushAllHeaderAndBodyToFile(filePath, &headerBuffer, &bodyBuffer);
}
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers();
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(),
true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */,
0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */);
if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) {
return false;
}
return dictBuffers.get()->flush(filePath);
}
/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) {
const int tmpFileNameBufSize = strlen(filePath)
@ -69,21 +84,21 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
FILE *const file = fopen(tmpFileName, "wb");
if (!file) {
AKLOGE("Dictionary file %s cannnot be opened.", tmpFileName);
AKLOGE("Dictionary file %s cannot be opened.", tmpFileName);
ASSERT(false);
return false;
}
// Write the dictionary header.
if (!writeBufferToFile(file, dictHeader)) {
remove(tmpFileName);
AKLOGE("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
AKLOGE("Dictionary header cannot be written. size: %d", dictHeader->getTailPosition());
ASSERT(false);
return false;
}
// Write the dictionary body.
if (!writeBufferToFile(file, dictBody)) {
remove(tmpFileName);
AKLOGE("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
AKLOGE("Dictionary body cannot be written. size: %d", dictBody->getTailPosition());
ASSERT(false);
return false;
}

3
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View File

@ -43,6 +43,9 @@ class DictFileWritingUtils {
static bool createEmptyV3DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool createEmptyV4DictFile(const char *const filePath,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool writeBufferToFile(FILE *const file,
const BufferWithExtendableBuffer *const buffer);
};