nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Concatenate dict buffers other than header to a single file."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2014-07-11 09:01:08 +00:00 committed by Android (Google) Code Review
commit 9e38fbaa2f
17 changed files with 234 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
View file

@ -17,6 +17,9 @@
#ifndef LATINIME_BIGRAM_DICT_CONTENT_H
#define LATINIME_BIGRAM_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
@ -27,12 +30,9 @@ namespace latinime {
class BigramDictContent : public SparseTableDictContent {
public:
BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo,
BigramDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool hasHistoricalInfo,
const bool isUpdatable)
: SparseTableDictContent(dictPath,
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
: SparseTableDictContent(buffers, bufferSizes, isUpdatable,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
mHasHistoricalInfo(hasHistoricalInfo) {}
@ -87,10 +87,8 @@ class BigramDictContent : public SparseTableDictContent {
return getUpdatableAddressLookupTable()->set(terminalId, bigramListPos);
}
bool flushToFile(const char *const dictPath) const {
return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
bool flushToFile(FILE *const file) const {
return flush(file);
}
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,

1
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/dict_content.h
View file

@ -24,7 +24,6 @@ namespace latinime {
class DictContent {
public:
virtual ~DictContent() {}
virtual bool isValid() const = 0;
protected:
DictContent() {}

7
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
View file

@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
return writeEntry(probabilityEntry, entryPos);
}
bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
bool ProbabilityDictContent::flushToFile(FILE *const file) const {
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
for (int i = 0; i < mSize; ++i) {
@ -81,10 +81,9 @@ bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
return false;
}
}
return probabilityDictContentToWrite.flush(dictPath,
Ver4DictConstants::FREQ_FILE_EXTENSION);
return probabilityDictContentToWrite.flush(file);
} else {
return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
return flush(file);
}
}

11
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h
View file

@ -17,6 +17,9 @@
#ifndef LATINIME_PROBABILITY_DICT_CONTENT_H
#define LATINIME_PROBABILITY_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
@ -29,9 +32,9 @@ class ProbabilityEntry;
class ProbabilityDictContent : public SingleDictContent {
public:
ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo,
const bool isUpdatable)
: SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
ProbabilityDictContent(uint8_t *const buffer, const int bufferSize,
const bool hasHistoricalInfo, const bool isUpdatable)
: SingleDictContent(buffer, bufferSize, isUpdatable),
mHasHistoricalInfo(hasHistoricalInfo),
mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
@ -42,7 +45,7 @@ class ProbabilityDictContent : public SingleDictContent {
bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const;
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ProbabilityDictContent *const originalProbabilityDictContent);

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
View file

@ -59,12 +59,6 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const {
return addressLookupTable->get(terminalId);
}
bool ShortcutDictContent::flushToFile(const char *const dictPath) const {
return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
}
bool ShortcutDictContent::runGC(
const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ShortcutDictContent *const originalShortcutDictContent) {

14
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
View file

@ -17,6 +17,9 @@
#ifndef LATINIME_SHORTCUT_DICT_CONTENT_H
#define LATINIME_SHORTCUT_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
@ -26,11 +29,8 @@ namespace latinime {
class ShortcutDictContent : public SparseTableDictContent {
public:
ShortcutDictContent(const char *const dictPath, const bool isUpdatable)
: SparseTableDictContent(dictPath,
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
ShortcutDictContent(uint8_t *const *buffers, const int *bufferSizes, const bool isUpdatable)
: SparseTableDictContent(buffers, bufferSizes, isUpdatable,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
@ -53,7 +53,9 @@ class ShortcutDictContent : public SparseTableDictContent {
// Returns head position of shortcut list for a PtNode specified by terminalId.
int getShortcutListHeadPos(const int terminalId) const;
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const {
return flush(file);
}
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ShortcutDictContent *const originalShortcutDictContent);

28
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View file

@ -17,35 +17,28 @@
#ifndef LATINIME_SINGLE_DICT_CONTENT_H
#define LATINIME_SINGLE_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
class SingleDictContent : public DictContent {
public:
SingleDictContent(const char *const dictPath, const char *const contentFileName,
const bool isUpdatable)
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mIsValid(mMmappedBuffer) {}
SingleDictContent(uint8_t *const buffer, const int bufferSize, const bool isUpdatable)
: mExpandableContentBuffer(buffer, bufferSize,
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
SingleDictContent()
: mMmappedBuffer(nullptr),
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mIsValid(true) {}
: mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE) {}
virtual ~SingleDictContent() {}
virtual bool isValid() const {
return mIsValid;
}
bool isNearSizeLimit() const {
return mExpandableContentBuffer.isNearSizeLimit();
}
@ -59,17 +52,14 @@ class SingleDictContent : public DictContent {
return &mExpandableContentBuffer;
}
bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const {
return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
contentFileNameSuffix, &mExpandableContentBuffer);
bool flush(FILE *const file) const {
return DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer);
}
private:
DISALLOW_COPY_AND_ASSIGN(SingleDictContent);
const MmappedBuffer::MmappedBufferPtr mMmappedBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SINGLE_DICT_CONTENT_H */

19
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
View file

@ -16,21 +16,22 @@
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
namespace latinime {
bool SparseTableDictContent::flush(const char *const dictPath,
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
const char *const contentFileNameSuffix) const {
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix,
&mExpandableLookupTableBuffer)){
const int SparseTableDictContent::LOOKUP_TABLE_BUFFER_INDEX = 0;
const int SparseTableDictContent::ADDRESS_TABLE_BUFFER_INDEX = 1;
const int SparseTableDictContent::CONTENT_BUFFER_INDEX = 2;
bool SparseTableDictContent::flush(FILE *const file) const {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableLookupTableBuffer)) {
return false;
}
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix,
&mExpandableAddressTableBuffer)) {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableAddressTableBuffer)) {
return false;
}
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix,
&mExpandableContentBuffer)) {
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableContentBuffer)) {
return false;
}
return true;

54
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h
View file

@ -17,12 +17,13 @@
#ifndef LATINIME_SPARSE_TABLE_DICT_CONTENT_H
#define LATINIME_SPARSE_TABLE_DICT_CONTENT_H
#include <cstdint>
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
namespace latinime {
@ -30,45 +31,29 @@ namespace latinime {
// TODO: Support multiple contents.
class SparseTableDictContent : public DictContent {
public:
AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName, const bool isUpdatable,
const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(
MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
mAddressTableBuffer(
MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
mContentBuffer(
MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
mExpandableLookupTableBuffer(
mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
AK_FORCE_INLINE SparseTableDictContent(uint8_t *const *buffers, const int *bufferSizes,
const bool isUpdatable, const int sparseTableBlockSize, const int sparseTableDataSize)
: mExpandableLookupTableBuffer(buffers[LOOKUP_TABLE_BUFFER_INDEX],
bufferSizes[LOOKUP_TABLE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableAddressTableBuffer(
mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
mExpandableAddressTableBuffer(buffers[ADDRESS_TABLE_BUFFER_INDEX],
bufferSizes[ADDRESS_TABLE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
mContentBuffer ? mContentBuffer->getBufferSize() : 0,
mExpandableContentBuffer(buffers[CONTENT_BUFFER_INDEX],
bufferSizes[CONTENT_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize),
mIsValid(mLookupTableBuffer && mAddressTableBuffer && mContentBuffer) {}
sparseTableBlockSize, sparseTableDataSize) {}
SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
: mLookupTableBuffer(), mAddressTableBuffer(), mContentBuffer(),
mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
: mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
sparseTableBlockSize, sparseTableDataSize), mIsValid(true) {}
sparseTableBlockSize, sparseTableDataSize) {}
virtual ~SparseTableDictContent() {}
virtual bool isValid() const {
return mIsValid;
}
bool isNearSizeLimit() const {
return mExpandableLookupTableBuffer.isNearSizeLimit()
|| mExpandableAddressTableBuffer.isNearSizeLimit()
@ -92,20 +77,19 @@ class SparseTableDictContent : public DictContent {
return &mExpandableContentBuffer;
}
bool flush(const char *const dictDirPath, const char *const lookupTableFileName,
const char *const addressTableFileName, const char *const contentFileName) const;
bool flush(FILE *const file) const;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTableDictContent);
const MmappedBuffer::MmappedBufferPtr mLookupTableBuffer;
const MmappedBuffer::MmappedBufferPtr mAddressTableBuffer;
const MmappedBuffer::MmappedBufferPtr mContentBuffer;
static const int LOOKUP_TABLE_BUFFER_INDEX;
static const int ADDRESS_TABLE_BUFFER_INDEX;
static const int CONTENT_BUFFER_INDEX;
BufferWithExtendableBuffer mExpandableLookupTableBuffer;
BufferWithExtendableBuffer mExpandableAddressTableBuffer;
BufferWithExtendableBuffer mExpandableContentBuffer;
SparseTable mAddressLookupTable;
const bool mIsValid;
};
} // namespace latinime
#endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */

7
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
View file

@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
}
bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const {
bool TerminalPositionLookupTable::flushToFile(FILE *const file) const {
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
// table and write the new table to the file.
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
@ -63,12 +63,11 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const
return false;
}
}
return lookupTableToWrite.flush(dictPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
return lookupTableToWrite.flush(file);
} else {
// We can simply use this lookup table because the buffer size has not been
// changed.
return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
return flush(file);
}
}

10
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
View file

@ -17,6 +17,8 @@
#ifndef LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H
#define LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H
#include <cstdint>
#include <cstdio>
#include <unordered_map>
#include "defines.h"
@ -29,9 +31,9 @@ class TerminalPositionLookupTable : public SingleDictContent {
public:
typedef std::unordered_map<int, int> TerminalIdMap;
TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable)
: SingleDictContent(dictPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
TerminalPositionLookupTable(uint8_t *const buffer, const int bufferSize,
const bool isUpdatable)
: SingleDictContent(buffer, bufferSize, isUpdatable),
mSize(getBuffer()->getTailPosition()
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
@ -45,7 +47,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
return mSize;
}
bool flushToFile(const char *const dictPath) const;
bool flushToFile(FILE *const file) const;
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);

132
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View file

@ -18,16 +18,19 @@
#include <cerrno>
#include <cstring>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <vector>
#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
namespace latinime {
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
const char *const dictPath, MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion) {
if (!headerBuffer) {
ASSERT(false);
@ -36,8 +39,32 @@ namespace latinime {
}
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
const bool isUpdatable = headerBuffer->isUpdatable();
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable,
formatVersion));
MmappedBuffer::MmappedBufferPtr bodyBuffer = MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::BODY_FILE_EXTENSION, isUpdatable);
if (!bodyBuffer) {
return Ver4DictBuffersPtr(nullptr);
}
std::vector<uint8_t *> buffers;
std::vector<int> bufferSizes;
uint8_t *const buffer = bodyBuffer->getBuffer();
int position = 0;
while (position < bodyBuffer->getBufferSize()) {
const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(buffer, &position);
buffers.push_back(buffer + position);
bufferSizes.push_back(bufferSize);
position += bufferSize;
if (bufferSize < 0 || position < 0 || position > bodyBuffer->getBufferSize()) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
}
if (buffers.size() != Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE) {
AKLOGE("The dict body file is corrupted.");
return Ver4DictBuffersPtr(nullptr);
}
return Ver4DictBuffersPtr(new Ver4DictBuffers(std::move(headerBuffer), std::move(bodyBuffer),
isUpdatable, formatVersion, buffers, bufferSizes));
}
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
@ -76,30 +103,32 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
Ver4DictConstants::HEADER_FILE_EXTENSION);
return false;
}
// Write trie file.
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION);
// Write body file.
const int bodyFilePathBufSize = FileUtils::getFilePathWithSuffixBufSize(dictPath,
Ver4DictConstants::BODY_FILE_EXTENSION);
char bodyFilePath[bodyFilePathBufSize];
FileUtils::getFilePathWithSuffix(dictPath, Ver4DictConstants::BODY_FILE_EXTENSION,
bodyFilePathBufSize, bodyFilePath);
const int fd = open(bodyFilePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
if (fd == -1) {
AKLOGE("File %s cannot be opened. errno: %d", bodyFilePath, errno);
ASSERT(false);
return false;
}
// Write dictionary contents.
if (!mTerminalPositionLookupTable.flushToFile(dictPath)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
FILE *const file = fdopen(fd, "wb");
if (!file) {
AKLOGE("fdopen failed for the file %s. errno: %d", filePath, errno);
ASSERT(false);
return false;
}
if (!mProbabilityDictContent.flushToFile(dictPath)) {
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mBigramDictContent.flushToFile(dictPath)) {
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mShortcutDictContent.flushToFile(dictPath)) {
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
if (!flushDictBuffers(file)) {
fclose(file);
return false;
}
fclose(file);
// Remove existing dictionary.
if (!FileUtils::removeDirAndFiles(dictDirPath)) {
AKLOGE("Existing directory %s cannot be removed.", dictDirPath);
@ -115,23 +144,60 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
return true;
}
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion)
: mHeaderBuffer(std::move(headerBuffer)),
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const {
// Write trie.
if (!DictFileWritingUtils::writeBufferToFileTail(file, &mExpandableTrieBuffer)) {
AKLOGE("Trie cannot be written. %s", tmpDirPath);
return false;
}
// Write terminal position lookup table.
if (!mTerminalPositionLookupTable.flushToFile(file)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
return false;
}
// Write probability dict content.
if (!mProbabilityDictContent.flushToFile(file)) {
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
return false;
}
// Write bigram dict content.
if (!mBigramDictContent.flushToFile(file)) {
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
return false;
}
// Write shortcut dict content.
if (!mShortcutDictContent.flushToFile(file)) {
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
return false;
}
return true;
}
Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
MmappedBuffer::MmappedBufferPtr &&bodyBuffer,
const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers, const std::vector<int> &contentBufferSizes)
: mHeaderBuffer(std::move(headerBuffer)), mDictBuffer(std::move(bodyBuffer)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
mDictBuffer ? mDictBuffer->getBufferSize() : 0,
mExpandableTrieBuffer(contentBuffers[Ver4DictConstants::TRIE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TRIE_BUFFER_INDEX],
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
mTerminalPositionLookupTable(dictPath, isUpdatable),
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mShortcutDictContent(dictPath, isUpdatable),
mTerminalPositionLookupTable(
contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX],
isUpdatable),
mProbabilityDictContent(
contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
mShortcutDictContent(&contentBuffers[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX], isUpdatable),
mIsUpdatable(isUpdatable) {}
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize)

17
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h
View file

@ -17,6 +17,7 @@
#ifndef LATINIME_VER4_DICT_BUFFER_H
#define LATINIME_VER4_DICT_BUFFER_H
#include <cstdio>
#include <memory>
#include "defines.h"
@ -36,7 +37,7 @@ class Ver4DictBuffers {
typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr;
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
MmappedBuffer::MmappedBufferPtr headerBuffer,
MmappedBuffer::MmappedBufferPtr &&headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion);
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
@ -45,9 +46,7 @@ class Ver4DictBuffers {
}
AK_FORCE_INLINE bool isValid() const {
return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid()
&& mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid()
&& mBigramDictContent.isValid() && mShortcutDictContent.isValid();
return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid();
}
AK_FORCE_INLINE bool isNearSizeLimit() const {
@ -120,12 +119,16 @@ class Ver4DictBuffers {
private:
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion);
Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
MmappedBuffer::MmappedBufferPtr &&bodyBuffer,
const bool isUpdatable, const FormatUtils::FORMAT_VERSION formatVersion,
const std::vector<uint8_t *> &contentBuffers,
const std::vector<int> &contentBufferSizes);
Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize);
bool flushDictBuffers(FILE *const file) const;
const MmappedBuffer::MmappedBufferPtr mHeaderBuffer;
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
const HeaderPolicy mHeaderPolicy;

31
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View file

@ -18,19 +18,8 @@
namespace latinime {
// These values MUST match the definitions in FormatSpec.java.
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
const char *const Ver4DictConstants::BODY_FILE_EXTENSION = ".body";
const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header";
const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq";
// tat = Terminal Address Table
const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
const char *const Ver4DictConstants::BIGRAM_FILE_EXTENSION = ".bigram_freq";
const char *const Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup";
const char *const Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION = ".bigram_index_freq";
const char *const Ver4DictConstants::SHORTCUT_FILE_EXTENSION = ".shortcut_shortcut";
const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".shortcut_lookup";
const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
".shortcut_index_shortcut";
// Version 4 dictionary size is implicitly limited to 8MB due to 3-byte offsets.
const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
@ -38,6 +27,21 @@ const int Ver4DictConstants::MAX_DICTIONARY_SIZE = 8 * 1024 * 1024;
// limited to 1MB to prevent from inefficient traversing.
const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
// NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie, TerminalAddressLookupTable and Probability.
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3
+ NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =
TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
const int Ver4DictConstants::PROBABILITY_SIZE = 1;
const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;
@ -67,4 +71,7 @@ const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3;
} // namespace latinime

23
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View file

@ -19,26 +19,26 @@
#include "defines.h"
#include <cstddef>
namespace latinime {
// TODO: Create PtConstants under the pt_common and move some constant values there.
// Note that there are corresponding definitions in FormatSpec.java.
class Ver4DictConstants {
public:
static const char *const TRIE_FILE_EXTENSION;
static const char *const BODY_FILE_EXTENSION;
static const char *const HEADER_FILE_EXTENSION;
static const char *const FREQ_FILE_EXTENSION;
static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_FILE_EXTENSION;
static const char *const BIGRAM_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_CONTENT_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_FILE_EXTENSION;
static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
static const int MAX_DICTIONARY_SIZE;
static const int MAX_DICT_EXTENDED_REGION_SIZE;
static const size_t NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE;
static const int TRIE_BUFFER_INDEX;
static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX;
static const int PROBABILITY_BUFFER_INDEX;
static const int BIGRAM_BUFFERS_INDEX;
static const int SHORTCUT_BUFFERS_INDEX;
static const int NOT_A_TERMINAL_ID;
static const int PROBABILITY_SIZE;
static const int FLAGS_IN_PROBABILITY_FILE_SIZE;
@ -68,6 +68,9 @@ class Ver4DictConstants {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
};
} // namespace latinime
#endif /* LATINIME_VER4_DICT_CONSTANTS_H */

20
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View file

@ -34,6 +34,8 @@
namespace latinime {
const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
// Enough size to describe buffer size.
const int DictFileWritingUtils::SIZE_OF_BUFFER_SIZE_FIELD = 4;
/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
const int dictVersion, const std::vector<int> localeAsCodePointVector,
@ -85,6 +87,18 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return flushBufferToFile(filePath, buffer);
}
/* static */ bool DictFileWritingUtils::writeBufferToFileTail(FILE *const file,
const BufferWithExtendableBuffer *const buffer) {
uint8_t bufferSize[SIZE_OF_BUFFER_SIZE_FIELD];
int writingPos = 0;
ByteArrayUtils::writeUintAndAdvancePosition(bufferSize, buffer->getTailPosition(),
SIZE_OF_BUFFER_SIZE_FIELD, &writingPos);
if (fwrite(bufferSize, SIZE_OF_BUFFER_SIZE_FIELD, 1 /* count */, file) < 1) {
return false;
}
return writeBufferToFile(file, buffer);
}
/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer) {
const int fd = open(filePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
@ -100,6 +114,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return false;
}
if (!writeBufferToFile(file, buffer)) {
fclose(file);
remove(filePath);
AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
buffer->getTailPosition());
@ -110,20 +125,17 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr>
return true;
}
// This closes file pointer when an error is caused and returns whether the writing was succeeded
// or not.
// Returns whether the writing was succeeded or not.
/* static */ bool DictFileWritingUtils::writeBufferToFile(FILE *const file,
const BufferWithExtendableBuffer *const buffer) {
const int originalBufSize = buffer->getOriginalBufferSize();
if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
originalBufSize, 1, file) < 1) {
fclose(file);
return false;
}
const int additionalBufSize = buffer->getUsedAdditionalBufferSize();
if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
additionalBufSize, 1, file) < 1) {
fclose(file);
return false;
}
return true;

5
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View file

@ -38,9 +38,14 @@ class DictFileWritingUtils {
static bool flushBufferToFileWithSuffix(const char *const basePath, const char *const suffix,
const BufferWithExtendableBuffer *const buffer);
static bool writeBufferToFileTail(FILE *const file,
const BufferWithExtendableBuffer *const buffer);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
static const int SIZE_OF_BUFFER_SIZE_FIELD;
static bool createEmptyV401DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,