nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Concatenate dict buffers other than header to a single file. 

Bug: 13664080
Change-Id: I34c9d8046b339c9b855be378a5fad907382d1359
main
Keisuke Kuroyanagi 2014-07-11 15:15:47 +09:00
parent b7ecb258dc
commit 2ac934296c
17 changed files with 234 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
View File

@ -17,6 +17,9 @@
#ifndef LATINIME_BIGRAM_DICT_CONTENT_H
#define LATINIME_BIGRAM_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
@ -27,12 +30,9 @@ namespace latinime {
class BigramDictContent : public SparseTableDictContent {
public:
BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo,
BigramDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool hasHistoricalInfo,
const bool isUpdatable)
: SparseTableDictContent(dictPath,
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
: SparseTableDictContent(buffers, bufferSizes, isUpdatable,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
mHasHistoricalInfo(hasHistoricalInfo) {}
@ -87,10 +87,8 @@ class BigramDictContent : public SparseTableDictContent {
return getUpdatableAddressLookupTable()->set(terminalId, bigramListPos);
}
bool flushToFile(const char *const dictPath) const {
return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
bool flushToFile(FILE *const file) const {
return flush(file);
}
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,

1
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
View File

@ -24,7 +24,6 @@ namespace latinime {
class DictContent {
public:
virtual ~DictContent() {}
virtual bool isValid() const = 0;
protected:
DictContent() {}

7
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
View File

@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
return writeEntry(probabilityEntry, entryPos);
}
bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
bool ProbabilityDictContent::flushToFile(FILE *const file) const {
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
for (int i = 0; i < mSize; ++i) {
@ -81,10 +81,9 @@ bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
return false;
}
}
return probabilityDictContentToWrite.flush(dictPath,
Ver4DictConstants::FREQ_FILE_EXTENSION);
return probabilityDictContentToWrite.flush(file);
} else {
return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
return flush(file);
}
}

11
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
View File

@ -17,6 +17,9 @@
#ifndef LATINIME_PROBABILITY_DICT_CONTENT_H
#define LATINIME_PROBABILITY_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
@ -29,9 +32,9 @@ class ProbabilityEntry;
class ProbabilityDictContent : public SingleDictContent {
public:
ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo,
const bool isUpdatable)
: SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
ProbabilityDictContent(uint8_t *const buffer, const int bufferSize,
const bool hasHistoricalInfo, const bool isUpdatable)
: SingleDictContent(buffer, bufferSize, isUpdatable),
mHasHistoricalInfo(hasHistoricalInfo),
mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
@ -42,7 +45,7 @@ class ProbabilityDictContent : public SingleDictContent {
bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const;
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ProbabilityDictContent *const originalProbabilityDictContent);

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
View File

@ -59,12 +59,6 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const {
return addressLookupTable->get(terminalId);
}
bool ShortcutDictContent::flushToFile(const char *const dictPath) const {
return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
}
bool ShortcutDictContent::runGC(
const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ShortcutDictContent *const originalShortcutDictContent) {

14
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
View File

@ -17,6 +17,9 @@
#ifndef LATINIME_SHORTCUT_DICT_CONTENT_H
#define LATINIME_SHORTCUT_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
@ -26,11 +29,8 @@ namespace latinime {
class ShortcutDictContent : public SparseTableDictContent {
public:
ShortcutDictContent(const char *const dictPath, const bool isUpdatable)
: SparseTableDictContent(dictPath,
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
ShortcutDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool isUpdatable)
: SparseTableDictContent(buffers, bufferSizes, isUpdatable,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
@ -53,7 +53,9 @@ class ShortcutDictContent : public SparseTableDictContent {
// Returns head position of shortcut list for a PtNode specified by terminalId.
int getShortcutListHeadPos(const int terminalId) const;
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const {
return flush(file);
}
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ShortcutDictContent *const originalShortcutDictContent);

28
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View File

@ -17,35 +17,28 @@
#ifndef LATINIME_SINGLE_DICT_CONTENT_H
#define LATINIME_SINGLE_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
class SingleDictContent : public DictContent {
public:
SingleDictContent(const char *const dictPath, const char *const contentFileName,
const bool isUpdatable)
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mIsValid(mMmappedBuffer) {}
SingleDictContent(uint8_t *const buffer, const int bufferSize, const bool isUpdatable)
: mExpandableContentBuffer(buffer, bufferSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
SingleDictContent()
: mMmappedBuffer(nullptr),
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mIsValid(true) {}
: mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE) {}
virtual ~SingleDictContent() {}
virtual bool isValid() const {
return mIsValid;
}
bool isNearSizeLimit() const {
return mExpandableContentBuffer.isNearSizeLimit();
}
@ -59,17 +52,14 @@ class SingleDictContent : public DictContent {
return &mExpandableContentBuffer;
}
bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const {
return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
contentFileNameSuffix, &mExpandableContentBuffer);
bool flush(FILE *const file) const {
return DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer);
}
private:
DISALLOW_COPY_AND_ASSIGN(SingleDictContent);
const MmappedBuffer::MmappedBufferPtr mMmappedBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SINGLE_DICT_CONTENT_H */

19
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
View File

@ -16,21 +16,22 @@
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
namespace latinime {
bool SparseTableDictContent::flush(const char *const dictPath,
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
const char *const contentFileNameSuffix) const {
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix,
&mExpandableLookupTableBuffer)){
const int SparseTableDictContent::LOOKUP_TABLE_BUFFER_INDEX = 0;
const int SparseTableDictContent::ADDRESS_TABLE_BUFFER_INDEX = 1;
const int SparseTableDictContent::CONTENT_BUFFER_INDEX = 2;
bool SparseTableDictContent::flush(FILE *const file) const {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableLookupTableBuffer)) {
return false;
}
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix,
&mExpandableAddressTableBuffer)) {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableAddressTableBuffer)) {
return false;
}
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix,
&mExpandableContentBuffer)) {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer)) {
return false;
}
return true;

54
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View File

@ -17,12 +17,13 @@
#ifndef LATINIME_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_SPARSE_TABLE_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
namespace latinime {
@ -30,45 +31,29 @@ namespace latinime {
// TODO: Support multiple contents.
class SparseTableDictContent : public DictContent {
public:
AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName, const bool isUpdatable,
const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(
MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
mAddressTableBuffer(
MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
mContentBuffer(
MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableLookupTableBuffer(
mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes,
const bool isUpdatable, const int sparseTableBlockSize, const int sparseTableDataSize)
: mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(
mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
mContentBuffer ? mContentBuffer->getBufferSize() : 0,
mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize),
mIsValid(mLookupTableBuffer && mAddressTableBuffer && mContentBuffer) {}
sparseTableBlockSize, sparseTableDataSize) {}
SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(), mAddressTableBuffer(), mContentBuffer(),
mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
: mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {}
sparseTableBlockSize, sparseTableDataSize) {}
virtual ~SparseTableDictContent() {}
virtual bool isValid() const {
return mIsValid;
}
bool isNearSizeLimit() const {
return mExpandableLookupTableBuffer.isNearSizeLimit()
|| mExpandableAddressTableBuffer.isNearSizeLimit()
@ -92,20 +77,19 @@ class SparseTableDictContent : public DictContent {
return &mExpandableContentBuffer;
}
bool flush(const char *const dictDirPath, const char *const lookupTableFileName,
const char *const addressTableFileName, const char *const contentFileName) const;
bool flush(FILE *const file) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent);
const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer;
const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer;
const MmappedBuffer::MmappedBufferPtr mContentBuffer;
static const int LOOKUP_TABLE_BUFFER_INDEX;
static const int ADDRESS_TABLE_BUFFER_INDEX;
static const int CONTENT_BUFFER_INDEX;
BufferWithExtendableBuffer mExpandableLookupTableBuffer;
BufferWithExtendableBuffer mExpandableAddressTableBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
SparseTable mAddressLookupTable;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */

7
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
View File

@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
}
bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const {
bool TerminalPositionLookupTable::flushToFile(FILE *const file) const {
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
// table and write the new table to the file.
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
@ -63,12 +63,11 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const
return false;
}
}
return lookupTableToWrite.flush(dictPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
return lookupTableToWrite.flush(file);
} else {
// We can simply use this lookup table because the buffer size has not been
// changed.
return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
return flush(file);
}
}

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
View File

@ -17,6 +17,8 @@
#ifndef LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H
#include <cstdint>
#include <cstdio>
#include <unordered_map>
#include "defines.h"
@ -29,9 +31,9 @@ class TerminalPositionLookupTable : public SingleDictContent {
public:
typedef std::unordered_map<int, int> TerminalIdMap;
TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable)
: SingleDictContent(dictPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
TerminalPositionLookupTable(uint8_t *const buffer, const int bufferSize,
const bool isUpdatable)
: SingleDictContent(buffer, bufferSize, isUpdatable),
mSize(getBuffer()->getTailPosition()
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
@ -45,7 +47,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
return mSize;
}
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const;
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);

132
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View File

@ -18,16 +18,19 @@
#include <cerrno>
#include <cstring>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
namespace latinime {
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
const char *const dictPath, MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion) {
if (!headerBuffer) {
ASSERT(false);
@ -36,8 +39,32 @@ namespace latinime {
}
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
const bool isUpdatable = headerBuffer->isUpdatable();
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable,
formatVersion));
MmappedBuffer::MmappedBufferPtr bodyBuffer = MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::BODY_FILE_EXTENSION, isUpdatable);
if (!bodyBuffer) {
return Ver4DictBuffersPtr(nullptr);
}
std::vector<uint8_t *> buffers;
std::vector<int> bufferSizes;
uint8_t *const buffer = bodyBuffer->getBuffer();
int position = 0;
while (position < bodyBuffer->getBufferSize()) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position);
buffers.push_back(buffer + position);
bufferSizes.push_back(bufferSize);
position += bufferSize;
if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
}
if (buffers.size() != Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
return Ver4DictBuffersPtr(new Ver4DictBuffers(std::move(headerBuffer), std::move(bodyBuffer),
isUpdatable, formatVersion, buffers, bufferSizes));
}
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
@ -76,30 +103,32 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
Ver4DictConstants::HEADER_FILE_EXTENSION);
return false;
}
// Write trie file.
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION);
// Write body file.
const int bodyFilePathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictPath,
Ver4DictConstants::BODY_FILE_EXTENSION);
char bodyFilePath[bodyFilePathBufSize];
FileUtils::getFilePathWithSuffix(dictPath, Ver4DictConstants::BODY_FILE_EXTENSION,
bodyFilePathBufSize, bodyFilePath);
const int fd = open(bodyFilePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
if (fd == -1) {
AKLOGE("File %s cannot be opened. errno: %d", bodyFilePath, errno);
ASSERT(false);
return false;
}
// Write dictionary contents.
if (!mTerminalPositionLookupTable.flushToFile(dictPath)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
FILE *const file = fdopen(fd, "wb");
if (!file) {
AKLOGE("fdopen failed for the file %s. errno: %d", filePath, errno);
ASSERT(false);
return false;
}
if (!mProbabilityDictContent.flushToFile(dictPath)) {
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mBigramDictContent.flushToFile(dictPath)) {
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mShortcutDictContent.flushToFile(dictPath)) {
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
if (!flushDictBuffers(file)) {
fclose(file);
return false;
}
fclose(file);
// Remove existing dictionary.
if (!FileUtils::removeDirAndFiles(dictDirPath)) {
AKLOGE("Existing directory %s cannot be removed.", dictDirPath);
@ -115,23 +144,60 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
return true;
}
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion)
: mHeaderBuffer(std::move(headerBuffer)),
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const {
// Write trie.
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableTrieBuffer)) {
AKLOGE("Trie cannot be written. %s", tmpDirPath);
return false;
}
// Write terminal position lookup table.
if (!mTerminalPositionLookupTable.flushToFile(file)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
return false;
}
// Write probability dict content.
if (!mProbabilityDictContent.flushToFile(file)) {
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
return false;
}
// Write bigram dict content.
if (!mBigramDictContent.flushToFile(file)) {
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
return false;
}
// Write shortcut dict content.
if (!mShortcutDictContent.flushToFile(file)) {
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
return false;
}
return true;
}
Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
MmappedBuffer::MmappedBufferPtr &&bodyBuffer,
const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers, const std::vector<int> &contentBufferSizes)
: mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
mDictBuffer ? mDictBuffer->getBufferSize() : 0,
mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(dictPath, isUpdatable),
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mShortcutDictContent(dictPath, isUpdatable),
mTerminalPositionLookupTable(
contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],
isUpdatable),
mProbabilityDictContent(
contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mShortcutDictContent(&contentBuffers[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX], isUpdatable),
mIsUpdatable(isUpdatable) {}
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize)

17
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View File

@ -17,6 +17,7 @@
#ifndef LATINIME_VER4_DICT_BUFFER_H
#define LATINIME_VER4_DICT_BUFFER_H
#include <cstdio>
#include <memory>
#include "defines.h"
@ -36,7 +37,7 @@ class Ver4DictBuffers {
typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr;
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
MmappedBuffer::MmappedBufferPtr headerBuffer,
MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion);
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
@ -45,9 +46,7 @@ class Ver4DictBuffers {
}
AK_FORCE_INLINE bool isValid() const {
return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid()
&& mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid()
&& mBigramDictContent.isValid() && mShortcutDictContent.isValid();
return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid();
}
AK_FORCE_INLINE bool isNearSizeLimit() const {
@ -120,12 +119,16 @@ class Ver4DictBuffers {
private:
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion);
Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
MmappedBuffer::MmappedBufferPtr &&bodyBuffer,
const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers,
const std::vector<int> &contentBufferSizes);
Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize);
bool flushDictBuffers(FILE *const file) const;
const MmappedBuffer::MmappedBufferPtr mHeaderBuffer;
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
const HeaderPolicy mHeaderPolicy;

31
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View File

@ -18,19 +18,8 @@
namespace latinime {
// These values MUST match the definitions in FormatSpec.java.
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
const char *const Ver4DictConstants::BODY_FILE_EXTENSION = ".body";
const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header";
const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq";
// tat = Terminal Address Table
const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
const char *const Ver4DictConstants::BIGRAM_FILE_EXTENSION = ".bigram_freq";
const char *const Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup";
const char *const Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION = ".bigram_index_freq";
const char *const Ver4DictConstants::SHORTCUT_FILE_EXTENSION = ".shortcut_shortcut";
const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".shortcut_lookup";
const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
".shortcut_index_shortcut";
// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
@ -38,6 +27,21 @@ const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
// limited to 1MB to prevent from inefficient traversing.
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie, TerminalAddressLookupTable and Probability.
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3
+ NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =
TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1;
const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;
@ -67,4 +71,7 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3;
} // namespace latinime

23
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -19,26 +19,26 @@
#include "defines.h"
#include <cstddef>
namespace latinime {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java.
class Ver4DictConstants {
public:
static const char *const TRIE_FILE_EXTENSION;
static const char *const BODY_FILE_EXTENSION;
static const char *const HEADER_FILE_EXTENSION;
static const char *const FREQ_FILE_EXTENSION;
static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_FILE_EXTENSION;
static const char *const BIGRAM_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_CONTENT_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_FILE_EXTENSION;
static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
static const int MAX_DICTIONARY_SIZE;
static const int MAX_DICT_EXTENDED_REGION_SIZE;
static const size_t NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE;
static const int TRIE_BUFFER_INDEX;
static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX;
static const int PROBABILITY_BUFFER_INDEX;
static const int BIGRAM_BUFFERS_INDEX;
static const int SHORTCUT_BUFFERS_INDEX;
static const int NOT_A_TERMINAL_ID;
static const int PROBABILITY_SIZE;
static const int FLAGS_IN_PROBABILITY_FILE_SIZE;
@ -68,6 +68,9 @@ class Ver4DictConstants {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
};
} // namespace latinime
#endif /* LATINIME_VER4_DICT_CONSTANTS_H */

20
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -34,6 +34,8 @@
namespace latinime {
const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
// Enough size to describe buffer size.
const int DictFileWritingUtils::SIZE_OF_BUFFER_SIZE_FIELD = 4;
/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
const int dictVersion, const std::vector<int> localeAsCodePointVector,
@ -85,6 +87,18 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return flushBufferToFile(filePath, buffer);
}
/* static */ bool DictFileWritingUtils::writeBufferToFileTail(FILE *const file,
const BufferWithExtendableBuffer *const buffer) {
uint8_t bufferSize[SIZE_OF_BUFFER_SIZE_FIELD];
int writingPos = 0;
ByteArrayUtils::writeUintAndAdvancePosition(bufferSize, buffer->getTailPosition(),
SIZE_OF_BUFFER_SIZE_FIELD, &writingPos);
if (fwrite(bufferSize, SIZE_OF_BUFFER_SIZE_FIELD, 1 /* count */, file) < 1) {
return false;
}
return writeBufferToFile(file, buffer);
}
/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer) {
const int fd = open(filePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
@ -100,6 +114,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return false;
}
if (!writeBufferToFile(file, buffer)) {
fclose(file);
remove(filePath);
AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
buffer->getTailPosition());
@ -110,20 +125,17 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return true;
}
// This closes file pointer when an error is caused and returns whether the writing was succeeded
// or not.
// Returns whether the writing was succeeded or not.
/* static */ bool DictFileWritingUtils::writeBufferToFile(FILE *const file,
const BufferWithExtendableBuffer *const buffer) {
const int originalBufSize = buffer->getOriginalBufferSize();
if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
originalBufSize, 1, file) < 1) {
fclose(file);
return false;
}
const int additionalBufSize = buffer->getUsedAdditionalBufferSize();
if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
additionalBufSize, 1, file) < 1) {
fclose(file);
return false;
}
return true;

5
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View File

@ -38,9 +38,14 @@ class DictFileWritingUtils {
static bool flushBufferToFileWithSuffix(const char *const basePath, const char *const suffix,
const BufferWithExtendableBuffer *const buffer);
static bool writeBufferToFileTail(FILE *const file,
const BufferWithExtendableBuffer *const buffer);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
static const int SIZE_OF_BUFFER_SIZE_FIELD;
static bool createEmptyV401DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,