From 2ac934296c0571ea252f3fb5a23fba29eb89c666 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Fri, 11 Jul 2014 15:15:47 +0900 Subject: [PATCH] Concatenate dict buffers other than header to a single file. Bug: 13664080 Change-Id: I34c9d8046b339c9b855be378a5fad907382d1359 --- .../v4/content/bigram_dict_content.h | 16 +-- .../structure/v4/content/dict_content.h | 1 - .../v4/content/probability_dict_content.cpp | 7 +- .../v4/content/probability_dict_content.h | 11 +- .../v4/content/shortcut_dict_content.cpp | 6 - .../v4/content/shortcut_dict_content.h | 14 +- .../v4/content/single_dict_content.h | 28 ++-- .../v4/content/sparse_table_dict_content.cpp | 19 +-- .../v4/content/sparse_table_dict_content.h | 54 +++---- .../terminal_position_lookup_table.cpp | 7 +- .../content/terminal_position_lookup_table.h | 10 +- .../structure/v4/ver4_dict_buffers.cpp | 132 +++++++++++++----- .../structure/v4/ver4_dict_buffers.h | 17 ++- .../structure/v4/ver4_dict_constants.cpp | 31 ++-- .../structure/v4/ver4_dict_constants.h | 23 +-- .../utils/dict_file_writing_utils.cpp | 20 ++- .../utils/dict_file_writing_utils.h | 5 + 17 files changed, 234 insertions(+), 167 deletions(-) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h index 033f18e9e..0ad619462 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h @@ -17,6 +17,9 @@ #ifndef LATINIME_BIGRAM_DICT_CONTENT_H #define LATINIME_BIGRAM_DICT_CONTENT_H +#include +#include + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h" @@ -27,12 +30,9 @@ namespace latinime { class BigramDictContent : public SparseTableDictContent { public: - BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo, + BigramDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool hasHistoricalInfo, const bool isUpdatable) - : SparseTableDictContent(dictPath, - Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, - Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, - Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable, + : SparseTableDictContent(buffers, bufferSizes, isUpdatable, Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE), mHasHistoricalInfo(hasHistoricalInfo) {} @@ -87,10 +87,8 @@ class BigramDictContent : public SparseTableDictContent { return getUpdatableAddressLookupTable()->set(terminalId, bigramListPos); } - bool flushToFile(const char *const dictPath) const { - return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, - Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, - Ver4DictConstants::BIGRAM_FILE_EXTENSION); + bool flushToFile(FILE *const file) const { + return flush(file); } bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h index 0c2f47073..c264aeac4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h @@ -24,7 +24,6 @@ namespace latinime { class DictContent { public: virtual ~DictContent() {} - virtual bool isValid() const = 0; protected: DictContent() {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp index 3b7c70efd..2425b3b2f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp @@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, return writeEntry(probabilityEntry, entryPos); } -bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { +bool ProbabilityDictContent::flushToFile(FILE *const file) const { if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo); for (int i = 0; i < mSize; ++i) { @@ -81,10 +81,9 @@ bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { return false; } } - return probabilityDictContentToWrite.flush(dictPath, - Ver4DictConstants::FREQ_FILE_EXTENSION); + return probabilityDictContentToWrite.flush(file); } else { - return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION); + return flush(file); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h index b065bc954..c4b7a901c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -17,6 +17,9 @@ #ifndef LATINIME_PROBABILITY_DICT_CONTENT_H #define LATINIME_PROBABILITY_DICT_CONTENT_H +#include +#include + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" @@ -29,9 +32,9 @@ class ProbabilityEntry; class ProbabilityDictContent : public SingleDictContent { public: - ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo, - const bool isUpdatable) - : SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable), + ProbabilityDictContent(uint8_t *const buffer, const int bufferSize, + const bool hasHistoricalInfo, const bool isUpdatable) + : SingleDictContent(buffer, bufferSize, isUpdatable), mHasHistoricalInfo(hasHistoricalInfo), mSize(getBuffer()->getTailPosition() / getEntrySize()) {} @@ -42,7 +45,7 @@ class ProbabilityDictContent : public SingleDictContent { bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry); - bool flushToFile(const char *const dictPath) const; + bool flushToFile(FILE *const file) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ProbabilityDictContent *const originalProbabilityDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp index 64d7bc0a5..41d9c544c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp @@ -59,12 +59,6 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const { return addressLookupTable->get(terminalId); } -bool ShortcutDictContent::flushToFile(const char *const dictPath) const { - return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, - Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, - Ver4DictConstants::SHORTCUT_FILE_EXTENSION); -} - bool ShortcutDictContent::runGC( const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ShortcutDictContent *const originalShortcutDictContent) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h index eaafc27bc..6a0e686ee 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h @@ -17,6 +17,9 @@ #ifndef LATINIME_SHORTCUT_DICT_CONTENT_H #define LATINIME_SHORTCUT_DICT_CONTENT_H +#include +#include + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" @@ -26,11 +29,8 @@ namespace latinime { class ShortcutDictContent : public SparseTableDictContent { public: - ShortcutDictContent(const char *const dictPath, const bool isUpdatable) - : SparseTableDictContent(dictPath, - Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, - Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, - Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable, + ShortcutDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool isUpdatable) + : SparseTableDictContent(buffers, bufferSizes, isUpdatable, Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {} @@ -53,7 +53,9 @@ class ShortcutDictContent : public SparseTableDictContent { // Returns head position of shortcut list for a PtNode specified by terminalId. int getShortcutListHeadPos(const int terminalId) const; - bool flushToFile(const char *const dictPath) const; + bool flushToFile(FILE *const file) const { + return flush(file); + } bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ShortcutDictContent *const originalShortcutDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index 215642234..7c5b69f5a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -17,35 +17,28 @@ #ifndef LATINIME_SINGLE_DICT_CONTENT_H #define LATINIME_SINGLE_DICT_CONTENT_H +#include +#include + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" -#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { class SingleDictContent : public DictContent { public: - SingleDictContent(const char *const dictPath, const char *const contentFileName, - const bool isUpdatable) - : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), - mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr, - mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0, - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mIsValid(mMmappedBuffer) {} + SingleDictContent(uint8_t *const buffer, const int bufferSize, const bool isUpdatable) + : mExpandableContentBuffer(buffer, bufferSize, + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {} SingleDictContent() - : mMmappedBuffer(nullptr), - mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mIsValid(true) {} + : mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE) {} virtual ~SingleDictContent() {} - virtual bool isValid() const { - return mIsValid; - } - bool isNearSizeLimit() const { return mExpandableContentBuffer.isNearSizeLimit(); } @@ -59,17 +52,14 @@ class SingleDictContent : public DictContent { return &mExpandableContentBuffer; } - bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const { - return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, - contentFileNameSuffix, &mExpandableContentBuffer); + bool flush(FILE *const file) const { + return DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer); } private: DISALLOW_COPY_AND_ASSIGN(SingleDictContent); - const MmappedBuffer::MmappedBufferPtr mMmappedBuffer; BufferWithExtendableBuffer mExpandableContentBuffer; - const bool mIsValid; }; } // namespace latinime #endif /* LATINIME_SINGLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp index 63c6ea3a4..896ce6bd2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp @@ -16,21 +16,22 @@ #include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h" +#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" + namespace latinime { -bool SparseTableDictContent::flush(const char *const dictPath, - const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix, - const char *const contentFileNameSuffix) const { - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix, - &mExpandableLookupTableBuffer)){ +const int SparseTableDictContent::LOOKUP_TABLE_BUFFER_INDEX = 0; +const int SparseTableDictContent::ADDRESS_TABLE_BUFFER_INDEX = 1; +const int SparseTableDictContent::CONTENT_BUFFER_INDEX = 2; + +bool SparseTableDictContent::flush(FILE *const file) const { + if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableLookupTableBuffer)) { return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix, - &mExpandableAddressTableBuffer)) { + if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableAddressTableBuffer)) { return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix, - &mExpandableContentBuffer)) { + if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer)) { return false; } return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h index fb6c88eef..a30ae6fe3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h @@ -17,12 +17,13 @@ #ifndef LATINIME_SPARSE_TABLE_DICT_CONTENT_H #define LATINIME_SPARSE_TABLE_DICT_CONTENT_H +#include +#include + #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" -#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" -#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" #include "suggest/policyimpl/dictionary/utils/sparse_table.h" namespace latinime { @@ -30,45 +31,29 @@ namespace latinime { // TODO: Support multiple contents. class SparseTableDictContent : public DictContent { public: - AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath, - const char *const lookupTableFileName, const char *const addressTableFileName, - const char *const contentFileName, const bool isUpdatable, - const int sparseTableBlockSize, const int sparseTableDataSize) - : mLookupTableBuffer( - MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)), - mAddressTableBuffer( - MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)), - mContentBuffer( - MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), - mExpandableLookupTableBuffer( - mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr, - mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0, + AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes, + const bool isUpdatable, const int sparseTableBlockSize, const int sparseTableDataSize) + : mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX], + bufferSizes[LOOKUP_TABLE_BUFFER_INDEX], BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableAddressTableBuffer( - mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr, - mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0, + mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX], + bufferSizes[ADDRESS_TABLE_BUFFER_INDEX], BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr, - mContentBuffer ? mContentBuffer->getBufferSize() : 0, + mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX], + bufferSizes[CONTENT_BUFFER_INDEX], BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, - sparseTableBlockSize, sparseTableDataSize), - mIsValid(mLookupTableBuffer && mAddressTableBuffer && mContentBuffer) {} + sparseTableBlockSize, sparseTableDataSize) {} SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize) - : mLookupTableBuffer(), mAddressTableBuffer(), mContentBuffer(), - mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + : mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer, - sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {} + sparseTableBlockSize, sparseTableDataSize) {} virtual ~SparseTableDictContent() {} - virtual bool isValid() const { - return mIsValid; - } - bool isNearSizeLimit() const { return mExpandableLookupTableBuffer.isNearSizeLimit() || mExpandableAddressTableBuffer.isNearSizeLimit() @@ -92,20 +77,19 @@ class SparseTableDictContent : public DictContent { return &mExpandableContentBuffer; } - bool flush(const char *const dictDirPath, const char *const lookupTableFileName, - const char *const addressTableFileName, const char *const contentFileName) const; + bool flush(FILE *const file) const; private: DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent); - const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer; - const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer; - const MmappedBuffer::MmappedBufferPtr mContentBuffer; + static const int LOOKUP_TABLE_BUFFER_INDEX; + static const int ADDRESS_TABLE_BUFFER_INDEX; + static const int CONTENT_BUFFER_INDEX; + BufferWithExtendableBuffer mExpandableLookupTableBuffer; BufferWithExtendableBuffer mExpandableAddressTableBuffer; BufferWithExtendableBuffer mExpandableContentBuffer; SparseTable mAddressLookupTable; - const bool mIsValid; }; } // namespace latinime #endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp index 0b17a009d..cf238ee5f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp @@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition( Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); } -bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const { +bool TerminalPositionLookupTable::flushToFile(FILE *const file) const { // If the used buffer size is smaller than the actual buffer size, regenerate the lookup // table and write the new table to the file. if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { @@ -63,12 +63,11 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const return false; } } - return lookupTableToWrite.flush(dictPath, - Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + return lookupTableToWrite.flush(file); } else { // We can simply use this lookup table because the buffer size has not been // changed. - return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + return flush(file); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index 816059560..c4c93f3f8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -17,6 +17,8 @@ #ifndef LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H #define LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H +#include +#include #include #include "defines.h" @@ -29,9 +31,9 @@ class TerminalPositionLookupTable : public SingleDictContent { public: typedef std::unordered_map TerminalIdMap; - TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable) - : SingleDictContent(dictPath, - Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), + TerminalPositionLookupTable(uint8_t *const buffer, const int bufferSize, + const bool isUpdatable) + : SingleDictContent(buffer, bufferSize, isUpdatable), mSize(getBuffer()->getTailPosition() / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {} @@ -45,7 +47,7 @@ class TerminalPositionLookupTable : public SingleDictContent { return mSize; } - bool flushToFile(const char *const dictPath) const; + bool flushToFile(FILE *const file) const; bool runGCTerminalIds(TerminalIdMap *const terminalIdMap); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index 5aa6b9a92..e4087be24 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -18,16 +18,19 @@ #include #include +#include #include #include +#include +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" namespace latinime { /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( - const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer, + const char *const dictPath, MmappedBuffer::MmappedBufferPtr &&headerBuffer, const FormatUtils::FORMAT_VERSION formatVersion) { if (!headerBuffer) { ASSERT(false); @@ -36,8 +39,32 @@ namespace latinime { } // TODO: take only dictDirPath, and open both header and trie files in the constructor below const bool isUpdatable = headerBuffer->isUpdatable(); - return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable, - formatVersion)); + + MmappedBuffer::MmappedBufferPtr bodyBuffer = MmappedBuffer::openBuffer(dictPath, + Ver4DictConstants::BODY_FILE_EXTENSION, isUpdatable); + if (!bodyBuffer) { + return Ver4DictBuffersPtr(nullptr); + } + std::vector buffers; + std::vector bufferSizes; + uint8_t *const buffer = bodyBuffer->getBuffer(); + int position = 0; + while (position < bodyBuffer->getBufferSize()) { + const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position); + buffers.push_back(buffer + position); + bufferSizes.push_back(bufferSize); + position += bufferSize; + if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) { + AKLOGE("The dict body file is corrupted."); + return Ver4DictBuffersPtr(nullptr); + } + } + if (buffers.size() != Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE) { + AKLOGE("The dict body file is corrupted."); + return Ver4DictBuffersPtr(nullptr); + } + return Ver4DictBuffersPtr(new Ver4DictBuffers(std::move(headerBuffer), std::move(bodyBuffer), + isUpdatable, formatVersion, buffers, bufferSizes)); } bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, @@ -76,30 +103,32 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, Ver4DictConstants::HEADER_FILE_EXTENSION); return false; } - // Write trie file. - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, - Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { - AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath, - Ver4DictConstants::TRIE_FILE_EXTENSION); + + // Write body file. + const int bodyFilePathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictPath, + Ver4DictConstants::BODY_FILE_EXTENSION); + char bodyFilePath[bodyFilePathBufSize]; + FileUtils::getFilePathWithSuffix(dictPath, Ver4DictConstants::BODY_FILE_EXTENSION, + bodyFilePathBufSize, bodyFilePath); + + const int fd = open(bodyFilePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); + if (fd == -1) { + AKLOGE("File %s cannot be opened. errno: %d", bodyFilePath, errno); + ASSERT(false); return false; } - // Write dictionary contents. - if (!mTerminalPositionLookupTable.flushToFile(dictPath)) { - AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); + FILE *const file = fdopen(fd, "wb"); + if (!file) { + AKLOGE("fdopen failed for the file %s. errno: %d", filePath, errno); + ASSERT(false); return false; } - if (!mProbabilityDictContent.flushToFile(dictPath)) { - AKLOGE("Probability dict content cannot be written. %s", tmpDirPath); - return false; - } - if (!mBigramDictContent.flushToFile(dictPath)) { - AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath); - return false; - } - if (!mShortcutDictContent.flushToFile(dictPath)) { - AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath); + + if (!flushDictBuffers(file)) { + fclose(file); return false; } + fclose(file); // Remove existing dictionary. if (!FileUtils::removeDirAndFiles(dictDirPath)) { AKLOGE("Existing directory %s cannot be removed.", dictDirPath); @@ -115,23 +144,60 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, return true; } -Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, - MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable, - const FormatUtils::FORMAT_VERSION formatVersion) - : mHeaderBuffer(std::move(headerBuffer)), - mDictBuffer(MmappedBuffer::openBuffer(dictPath, - Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), +bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const { + // Write trie. + if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableTrieBuffer)) { + AKLOGE("Trie cannot be written. %s", tmpDirPath); + return false; + } + // Write terminal position lookup table. + if (!mTerminalPositionLookupTable.flushToFile(file)) { + AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); + return false; + } + // Write probability dict content. + if (!mProbabilityDictContent.flushToFile(file)) { + AKLOGE("Probability dict content cannot be written. %s", tmpDirPath); + return false; + } + // Write bigram dict content. + if (!mBigramDictContent.flushToFile(file)) { + AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath); + return false; + } + // Write shortcut dict content. + if (!mShortcutDictContent.flushToFile(file)) { + AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath); + return false; + } + return true; +} + +Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, + MmappedBuffer::MmappedBufferPtr &&bodyBuffer, + const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion, + const std::vector &contentBuffers, const std::vector &contentBufferSizes) + : mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)), mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion), mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr, mHeaderPolicy.getSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr, - mDictBuffer ? mDictBuffer->getBufferSize() : 0, + mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX], + contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX], BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mTerminalPositionLookupTable(dictPath, isUpdatable), - mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mShortcutDictContent(dictPath, isUpdatable), + mTerminalPositionLookupTable( + contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX], + contentBufferSizes[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX], + isUpdatable), + mProbabilityDictContent( + contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], + contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], + mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), + mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], + &contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], + mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), + mShortcutDictContent(&contentBuffers[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX], + &contentBufferSizes[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX], isUpdatable), mIsUpdatable(isUpdatable) {} Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index df177c14a..d2f4f2aed 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -17,6 +17,7 @@ #ifndef LATINIME_VER4_DICT_BUFFER_H #define LATINIME_VER4_DICT_BUFFER_H +#include #include #include "defines.h" @@ -36,7 +37,7 @@ class Ver4DictBuffers { typedef std::unique_ptr Ver4DictBuffersPtr; static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, - MmappedBuffer::MmappedBufferPtr headerBuffer, + MmappedBuffer::MmappedBufferPtr &&headerBuffer, const FormatUtils::FORMAT_VERSION formatVersion); static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( @@ -45,9 +46,7 @@ class Ver4DictBuffers { } AK_FORCE_INLINE bool isValid() const { - return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid() - && mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid() - && mBigramDictContent.isValid() && mShortcutDictContent.isValid(); + return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid(); } AK_FORCE_INLINE bool isNearSizeLimit() const { @@ -120,12 +119,16 @@ class Ver4DictBuffers { private: DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); - Ver4DictBuffers(const char *const dictDirPath, - const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable, - const FormatUtils::FORMAT_VERSION formatVersion); + Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, + MmappedBuffer::MmappedBufferPtr &&bodyBuffer, + const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion, + const std::vector &contentBuffers, + const std::vector &contentBufferSizes); Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize); + bool flushDictBuffers(FILE *const file) const; + const MmappedBuffer::MmappedBufferPtr mHeaderBuffer; const MmappedBuffer::MmappedBufferPtr mDictBuffer; const HeaderPolicy mHeaderPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index 345cabbf9..d45dfe377 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -18,19 +18,8 @@ namespace latinime { -// These values MUST match the definitions in FormatSpec.java. -const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie"; +const char *const Ver4DictConstants::BODY_FILE_EXTENSION = ".body"; const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header"; -const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq"; -// tat = Terminal Address Table -const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; -const char *const Ver4DictConstants::BIGRAM_FILE_EXTENSION = ".bigram_freq"; -const char *const Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup"; -const char *const Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION = ".bigram_index_freq"; -const char *const Ver4DictConstants::SHORTCUT_FILE_EXTENSION = ".shortcut_shortcut"; -const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".shortcut_lookup"; -const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION = - ".shortcut_index_shortcut"; // Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets. const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024; @@ -38,6 +27,21 @@ const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024; // limited to 1MB to prevent from inefficient traversing. const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024; +// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie, TerminalAddressLookupTable and Probability. +// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut. +const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE = + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3 + + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2; +const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0; +const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX = + TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; +const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX = + TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; +const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX = + PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; +const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX = + BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; + const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1; const int Ver4DictConstants::PROBABILITY_SIZE = 1; const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1; @@ -67,4 +71,7 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1; const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F; const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80; +const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1; +const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3; + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index b4effca9c..e8f6739ba 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -19,26 +19,26 @@ #include "defines.h" +#include + namespace latinime { // TODO: Create PtConstants under the pt_common and move some constant values there. // Note that there are corresponding definitions in FormatSpec.java. class Ver4DictConstants { public: - static const char *const TRIE_FILE_EXTENSION; + static const char *const BODY_FILE_EXTENSION; static const char *const HEADER_FILE_EXTENSION; - static const char *const FREQ_FILE_EXTENSION; - static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION; - static const char *const BIGRAM_FILE_EXTENSION; - static const char *const BIGRAM_LOOKUP_TABLE_FILE_EXTENSION; - static const char *const BIGRAM_CONTENT_TABLE_FILE_EXTENSION; - static const char *const SHORTCUT_FILE_EXTENSION; - static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION; - static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION; - static const int MAX_DICTIONARY_SIZE; static const int MAX_DICT_EXTENDED_REGION_SIZE; + static const size_t NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE; + static const int TRIE_BUFFER_INDEX; + static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX; + static const int PROBABILITY_BUFFER_INDEX; + static const int BIGRAM_BUFFERS_INDEX; + static const int SHORTCUT_BUFFERS_INDEX; + static const int NOT_A_TERMINAL_ID; static const int PROBABILITY_SIZE; static const int FLAGS_IN_PROBABILITY_FILE_SIZE; @@ -68,6 +68,9 @@ class Ver4DictConstants { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants); + + static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; + static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; }; } // namespace latinime #endif /* LATINIME_VER4_DICT_CONSTANTS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index 1d202c3dd..b7e2a7278 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -34,6 +34,8 @@ namespace latinime { const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp"; +// Enough size to describe buffer size. +const int DictFileWritingUtils::SIZE_OF_BUFFER_SIZE_FIELD = 4; /* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath, const int dictVersion, const std::vector localeAsCodePointVector, @@ -85,6 +87,18 @@ template return flushBufferToFile(filePath, buffer); } +/* static */ bool DictFileWritingUtils::writeBufferToFileTail(FILE *const file, + const BufferWithExtendableBuffer *const buffer) { + uint8_t bufferSize[SIZE_OF_BUFFER_SIZE_FIELD]; + int writingPos = 0; + ByteArrayUtils::writeUintAndAdvancePosition(bufferSize, buffer->getTailPosition(), + SIZE_OF_BUFFER_SIZE_FIELD, &writingPos); + if (fwrite(bufferSize, SIZE_OF_BUFFER_SIZE_FIELD, 1 /* count */, file) < 1) { + return false; + } + return writeBufferToFile(file, buffer); +} + /* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath, const BufferWithExtendableBuffer *const buffer) { const int fd = open(filePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR); @@ -100,6 +114,7 @@ template return false; } if (!writeBufferToFile(file, buffer)) { + fclose(file); remove(filePath); AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath, buffer->getTailPosition()); @@ -110,20 +125,17 @@ template return true; } -// This closes file pointer when an error is caused and returns whether the writing was succeeded -// or not. +// Returns whether the writing was succeeded or not. /* static */ bool DictFileWritingUtils::writeBufferToFile(FILE *const file, const BufferWithExtendableBuffer *const buffer) { const int originalBufSize = buffer->getOriginalBufferSize(); if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), originalBufSize, 1, file) < 1) { - fclose(file); return false; } const int additionalBufSize = buffer->getUsedAdditionalBufferSize(); if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), additionalBufSize, 1, file) < 1) { - fclose(file); return false; } return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h index 0dd1256ee..4843b3b32 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -38,9 +38,14 @@ class DictFileWritingUtils { static bool flushBufferToFileWithSuffix(const char *const basePath, const char *const suffix, const BufferWithExtendableBuffer *const buffer); + static bool writeBufferToFileTail(FILE *const file, + const BufferWithExtendableBuffer *const buffer); + private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils); + static const int SIZE_OF_BUFFER_SIZE_FIELD; + static bool createEmptyV401DictFile(const char *const filePath, const std::vector localeAsCodePointVector, const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,