Quit passing header file path to native.
Bug: 12089197 Change-Id: Ia898f8d2836c0c131a1834dc0a396d9121ecb972main
parent
4e9d115401
commit
9ff51527a3
|
@ -60,10 +60,9 @@ abstract public class AbstractDictionaryWriter {
|
||||||
abstract protected void writeDictionary(final DictEncoder dictEncoder,
|
abstract protected void writeDictionary(final DictEncoder dictEncoder,
|
||||||
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
|
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
|
||||||
|
|
||||||
public void write(final String fileName, final Map<String, String> attributeMap) {
|
public void write(final File file, final Map<String, String> attributeMap) {
|
||||||
final String tempFileName = fileName + ".temp";
|
final String tempFilePath = file.getAbsolutePath() + ".temp";
|
||||||
final File file = new File(mContext.getFilesDir(), fileName);
|
final File tempFile = new File(tempFilePath);
|
||||||
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
|
|
||||||
try {
|
try {
|
||||||
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
|
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
|
||||||
writeDictionary(dictEncoder, attributeMap);
|
writeDictionary(dictEncoder, attributeMap);
|
||||||
|
|
|
@ -144,20 +144,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
return mBinaryDictionary.isValidDictionary();
|
return mBinaryDictionary.isValidDictionary();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String getFileNameToCreateDict(final String dictName) {
|
private File getDictFile() {
|
||||||
return dictName + DICT_FILE_EXTENSION;
|
return new File(mContext.getFilesDir(), mDictName + DICT_FILE_EXTENSION);
|
||||||
}
|
|
||||||
|
|
||||||
protected String getFileNameToOpenDict(final String dictName) {
|
|
||||||
return getFileNameToCreateDict(dictName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private File getFileToCreateDict() {
|
|
||||||
return new File(mContext.getFilesDir(), getFileNameToCreateDict(mDictName));
|
|
||||||
}
|
|
||||||
|
|
||||||
private File getFileToOpenDict() {
|
|
||||||
return new File(mContext.getFilesDir(), getFileNameToOpenDict(mDictName));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -270,17 +258,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
public void run() {
|
public void run() {
|
||||||
if (mDictionaryWriter == null) {
|
if (mDictionaryWriter == null) {
|
||||||
mBinaryDictionary.close();
|
mBinaryDictionary.close();
|
||||||
final File file = getFileToCreateDict();
|
final File file = getDictFile();
|
||||||
file.delete();
|
if (file.exists() && !FileUtils.deleteRecursively(file)) {
|
||||||
|
Log.e(TAG, "Can't remove a file: " + file.getName());
|
||||||
|
}
|
||||||
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
|
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
|
||||||
// We have 'fileToOpen' in addition to 'file' for the v4 dictionary format
|
|
||||||
// where 'file' is a directory, and 'fileToOpen' is a normal file.
|
|
||||||
final File fileToOpen = getFileToOpenDict();
|
|
||||||
// TODO: Make BinaryDictionary's constructor be able to accept filename
|
|
||||||
// without extension.
|
|
||||||
mBinaryDictionary = new BinaryDictionary(
|
mBinaryDictionary = new BinaryDictionary(
|
||||||
fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(),
|
file.getAbsolutePath(), 0 /* offset */, file.length(),
|
||||||
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
|
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
|
||||||
} else {
|
} else {
|
||||||
mDictionaryWriter.clear();
|
mDictionaryWriter.clear();
|
||||||
|
@ -531,7 +516,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
final File file = getFileToOpenDict();
|
final File file = getDictFile();
|
||||||
final String filename = file.getAbsolutePath();
|
final String filename = file.getAbsolutePath();
|
||||||
final long length = file.length();
|
final long length = file.length();
|
||||||
|
|
||||||
|
@ -572,14 +557,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
if (needsToReloadBeforeWriting()) {
|
if (needsToReloadBeforeWriting()) {
|
||||||
mDictionaryWriter.clear();
|
mDictionaryWriter.clear();
|
||||||
loadDictionaryAsync();
|
loadDictionaryAsync();
|
||||||
mDictionaryWriter.write(getFileNameToCreateDict(mDictName), getHeaderAttributeMap());
|
mDictionaryWriter.write(getDictFile(), getHeaderAttributeMap());
|
||||||
} else {
|
} else {
|
||||||
if (mBinaryDictionary == null || !isValidDictionary()
|
if (mBinaryDictionary == null || !isValidDictionary()
|
||||||
// TODO: remove the check below
|
// TODO: remove the check below
|
||||||
|| !matchesExpectedBinaryDictFormatVersionForThisType(
|
|| !matchesExpectedBinaryDictFormatVersionForThisType(
|
||||||
mBinaryDictionary.getFormatVersion())) {
|
mBinaryDictionary.getFormatVersion())) {
|
||||||
final File file = getFileToCreateDict();
|
final File file = getDictFile();
|
||||||
if (!FileUtils.deleteRecursively(file)) {
|
if (file.exists() && !FileUtils.deleteRecursively(file)) {
|
||||||
Log.e(TAG, "Can't remove a file: " + file.getName());
|
Log.e(TAG, "Can't remove a file: " + file.getName());
|
||||||
}
|
}
|
||||||
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
|
@ -706,8 +691,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
|
|
||||||
// TODO: cache the file's existence so that we avoid doing a disk access each time.
|
// TODO: cache the file's existence so that we avoid doing a disk access each time.
|
||||||
private boolean dictionaryFileExists() {
|
private boolean dictionaryFileExists() {
|
||||||
final File file = getFileToOpenDict();
|
return getDictFile().exists();
|
||||||
return file.exists();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -112,16 +112,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
||||||
return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION;
|
return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
protected String getFileNameToCreateDict(final String dictName) {
|
|
||||||
return dictName;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected String getFileNameToOpenDict(final String dictName) {
|
|
||||||
return dictName + "/" + dictName + FormatSpec.HEADER_FILE_EXTENSION;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addMultipleDictionaryEntriesToDictionary(
|
public void addMultipleDictionaryEntriesToDictionary(
|
||||||
final ArrayList<LanguageModelParam> languageModelParams,
|
final ArrayList<LanguageModelParam> languageModelParams,
|
||||||
final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
|
final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
|
||||||
|
|
|
@ -16,11 +16,13 @@
|
||||||
|
|
||||||
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
|
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
|
||||||
|
|
||||||
|
#include <climits>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
|
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
|
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
|
||||||
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h"
|
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
|
#include "suggest/policyimpl/dictionary/utils/file_utils.h"
|
||||||
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
|
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
|
||||||
|
@ -32,41 +34,99 @@ namespace latinime {
|
||||||
DictionaryStructureWithBufferPolicyFactory
|
DictionaryStructureWithBufferPolicyFactory
|
||||||
::newDictionaryStructureWithBufferPolicy(const char *const path,
|
::newDictionaryStructureWithBufferPolicy(const char *const path,
|
||||||
const int bufOffset, const int size, const bool isUpdatable) {
|
const int bufOffset, const int size, const bool isUpdatable) {
|
||||||
// Allocated buffer in MmapedBuffer::newBuffer() will be freed in the destructor of
|
if (FileUtils::existsDir(path)) {
|
||||||
// MmappedBufferWrapper if the instance has the responsibility.
|
// Given path represents a directory.
|
||||||
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size,
|
return newPolicyforDirectoryDict(path, isUpdatable);
|
||||||
|
} else {
|
||||||
|
if (isUpdatable) {
|
||||||
|
AKLOGE("One file dictionaries don't support updating. path: %s", path);
|
||||||
|
ASSERT(false);
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
|
}
|
||||||
|
return newPolicyforFileDict(path, bufOffset, size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
|
DictionaryStructureWithBufferPolicyFactory::newPolicyforDirectoryDict(
|
||||||
|
const char *const path, const bool isUpdatable) {
|
||||||
|
const int headerFilePathBufSize = PATH_MAX + 1 /* terminator */;
|
||||||
|
char headerFilePath[headerFilePathBufSize];
|
||||||
|
getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath);
|
||||||
|
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
||||||
|
// MmappedBufferPtr if the instance has the responsibility.
|
||||||
|
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(headerFilePath,
|
||||||
isUpdatable);
|
isUpdatable);
|
||||||
if (!mmappedBuffer.get()) {
|
if (!mmappedBuffer.get()) {
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
}
|
}
|
||||||
|
switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(),
|
||||||
|
mmappedBuffer.get()->getBufferSize())) {
|
||||||
|
case FormatUtils::VERSION_2:
|
||||||
|
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
|
||||||
|
break;
|
||||||
|
case FormatUtils::VERSION_4: {
|
||||||
|
const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
|
||||||
|
char dictPath[dictDirPathBufSize];
|
||||||
|
if (!FileUtils::getFilePathWithoutSuffix(headerFilePath,
|
||||||
|
Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
|
||||||
|
AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
|
||||||
|
ASSERT(false);
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
|
}
|
||||||
|
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
|
||||||
|
Ver4DictBuffers::openVer4DictBuffers(dictPath, mmappedBuffer);
|
||||||
|
if (!dictBuffers.get()->isValid()) {
|
||||||
|
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
|
||||||
|
path);
|
||||||
|
ASSERT(false);
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
|
}
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
||||||
|
new Ver4PatriciaTriePolicy(dictBuffers));
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
ASSERT(false);
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
|
DictionaryStructureWithBufferPolicyFactory::newPolicyforFileDict(
|
||||||
|
const char *const path, const int bufOffset, const int size) {
|
||||||
|
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
||||||
|
// MmappedBufferPtr if the instance has the responsibility.
|
||||||
|
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset,
|
||||||
|
size, false /* isUpdatable */);
|
||||||
|
if (!mmappedBuffer.get()) {
|
||||||
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
|
}
|
||||||
switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(),
|
switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(),
|
||||||
mmappedBuffer.get()->getBufferSize())) {
|
mmappedBuffer.get()->getBufferSize())) {
|
||||||
case FormatUtils::VERSION_2:
|
case FormatUtils::VERSION_2:
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
||||||
new PatriciaTriePolicy(mmappedBuffer));
|
new PatriciaTriePolicy(mmappedBuffer));
|
||||||
case FormatUtils::VERSION_4: {
|
case FormatUtils::VERSION_4:
|
||||||
const int dictDirPathBufSize = strlen(path) + 1 /* terminator */;
|
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
|
||||||
char dictDirPath[dictDirPathBufSize];
|
break;
|
||||||
if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION,
|
|
||||||
dictDirPathBufSize, dictDirPath)) {
|
|
||||||
// Dictionary file name is not valid as a version 4 dictionary.
|
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
|
||||||
}
|
|
||||||
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
|
|
||||||
Ver4DictBuffers::openVer4DictBuffers(dictDirPath, mmappedBuffer);
|
|
||||||
if (!dictBuffers.get()->isValid()) {
|
|
||||||
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements.");
|
|
||||||
ASSERT(false);
|
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
|
||||||
}
|
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
|
||||||
new Ver4PatriciaTriePolicy(dictBuffers));
|
|
||||||
}
|
|
||||||
default:
|
default:
|
||||||
AKLOGE("DICT: dictionary format is unknown, bad magic number");
|
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
|
||||||
|
break;
|
||||||
|
}
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir(
|
||||||
|
const char *const dictDirPath, const int outHeaderFileBufSize,
|
||||||
|
char *const outHeaderFilePath) {
|
||||||
|
const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */;
|
||||||
|
char dictName[dictNameBufSize];
|
||||||
|
FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName);
|
||||||
|
snprintf(outHeaderFilePath, outHeaderFileBufSize, "%s/%s%s", dictDirPath,
|
||||||
|
dictName, Ver4DictConstants::HEADER_FILE_EXTENSION);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
|
|
|
@ -33,6 +33,15 @@ class DictionaryStructureWithBufferPolicyFactory {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
|
||||||
|
|
||||||
|
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
|
newPolicyforDirectoryDict(const char *const path, const bool isUpdatable);
|
||||||
|
|
||||||
|
static DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
|
newPolicyforFileDict(const char *const path, const int bufOffset, const int size);
|
||||||
|
|
||||||
|
static void getHeaderFilePathInDictDir(const char *const dirPath,
|
||||||
|
const int outHeaderFileBufSize, char *const outHeaderFilePath);
|
||||||
};
|
};
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
#endif // LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H
|
#endif // LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H
|
||||||
|
|
|
@ -27,9 +27,9 @@ namespace latinime {
|
||||||
|
|
||||||
class BigramDictContent : public SparseTableDictContent {
|
class BigramDictContent : public SparseTableDictContent {
|
||||||
public:
|
public:
|
||||||
BigramDictContent(const char *const dictDirPath, const bool hasHistoricalInfo,
|
BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo,
|
||||||
const bool isUpdatable)
|
const bool isUpdatable)
|
||||||
: SparseTableDictContent(dictDirPath,
|
: SparseTableDictContent(dictPath,
|
||||||
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
|
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
|
||||||
|
@ -73,8 +73,8 @@ class BigramDictContent : public SparseTableDictContent {
|
||||||
|
|
||||||
bool copyBigramList(const int bigramListPos, const int toPos);
|
bool copyBigramList(const int bigramListPos, const int toPos);
|
||||||
|
|
||||||
bool flushToFile(const char *const dictBasePath) const {
|
bool flushToFile(const char *const dictPath) const {
|
||||||
return flush(dictBasePath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
|
return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
|
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
|
||||||
}
|
}
|
||||||
|
|
|
@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
|
||||||
return writeEntry(probabilityEntry, entryPos);
|
return writeEntry(probabilityEntry, entryPos);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const {
|
bool ProbabilityDictContent::flushToFile(const char *const dictPath) const {
|
||||||
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
|
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
|
||||||
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
|
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
|
||||||
for (int i = 0; i < mSize; ++i) {
|
for (int i = 0; i < mSize; ++i) {
|
||||||
|
@ -81,10 +81,10 @@ bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return probabilityDictContentToWrite.flush(dictBasePath,
|
return probabilityDictContentToWrite.flush(dictPath,
|
||||||
Ver4DictConstants::FREQ_FILE_EXTENSION);
|
Ver4DictConstants::FREQ_FILE_EXTENSION);
|
||||||
} else {
|
} else {
|
||||||
return flush(dictBasePath, Ver4DictConstants::FREQ_FILE_EXTENSION);
|
return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -29,9 +29,9 @@ class ProbabilityEntry;
|
||||||
|
|
||||||
class ProbabilityDictContent : public SingleDictContent {
|
class ProbabilityDictContent : public SingleDictContent {
|
||||||
public:
|
public:
|
||||||
ProbabilityDictContent(const char *const dictDirPath, const bool hasHistoricalInfo,
|
ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo,
|
||||||
const bool isUpdatable)
|
const bool isUpdatable)
|
||||||
: SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
|
: SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable),
|
||||||
mHasHistoricalInfo(hasHistoricalInfo),
|
mHasHistoricalInfo(hasHistoricalInfo),
|
||||||
mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
|
mSize(getBuffer()->getTailPosition() / getEntrySize()) {}
|
||||||
|
|
||||||
|
@ -42,7 +42,7 @@ class ProbabilityDictContent : public SingleDictContent {
|
||||||
|
|
||||||
bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
|
bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry);
|
||||||
|
|
||||||
bool flushToFile(const char *const dictDirPath) const;
|
bool flushToFile(const char *const dictPath) const;
|
||||||
|
|
||||||
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
|
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
|
||||||
const ProbabilityDictContent *const originalProbabilityDictContent);
|
const ProbabilityDictContent *const originalProbabilityDictContent);
|
||||||
|
|
|
@ -46,8 +46,8 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const {
|
||||||
return addressLookupTable->get(terminalId);
|
return addressLookupTable->get(terminalId);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ShortcutDictContent::flushToFile(const char *const dictBasePath) const {
|
bool ShortcutDictContent::flushToFile(const char *const dictPath) const {
|
||||||
return flush(dictBasePath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
|
return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
|
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,8 +26,8 @@ namespace latinime {
|
||||||
|
|
||||||
class ShortcutDictContent : public SparseTableDictContent {
|
class ShortcutDictContent : public SparseTableDictContent {
|
||||||
public:
|
public:
|
||||||
ShortcutDictContent(const char *const dictDirPath, const bool isUpdatable)
|
ShortcutDictContent(const char *const dictPath, const bool isUpdatable)
|
||||||
: SparseTableDictContent(dictDirPath,
|
: SparseTableDictContent(dictPath,
|
||||||
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
|
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
|
||||||
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
|
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
|
||||||
|
@ -53,7 +53,7 @@ class ShortcutDictContent : public SparseTableDictContent {
|
||||||
// Returns head position of shortcut list for a PtNode specified by terminalId.
|
// Returns head position of shortcut list for a PtNode specified by terminalId.
|
||||||
int getShortcutListHeadPos(const int terminalId) const;
|
int getShortcutListHeadPos(const int terminalId) const;
|
||||||
|
|
||||||
bool flushToFile(const char *const dictBasePath) const;
|
bool flushToFile(const char *const dictPath) const;
|
||||||
|
|
||||||
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
|
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
|
||||||
const ShortcutDictContent *const originalShortcutDictContent);
|
const ShortcutDictContent *const originalShortcutDictContent);
|
||||||
|
|
|
@ -28,9 +28,9 @@ namespace latinime {
|
||||||
|
|
||||||
class SingleDictContent : public DictContent {
|
class SingleDictContent : public DictContent {
|
||||||
public:
|
public:
|
||||||
SingleDictContent(const char *const dictDirPath, const char *const contentFileName,
|
SingleDictContent(const char *const dictPath, const char *const contentFileName,
|
||||||
const bool isUpdatable)
|
const bool isUpdatable)
|
||||||
: mMmappedBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)),
|
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
||||||
mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0,
|
mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0,
|
||||||
mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0,
|
mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0,
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
|
@ -59,8 +59,8 @@ class SingleDictContent : public DictContent {
|
||||||
return &mExpandableContentBuffer;
|
return &mExpandableContentBuffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool flush(const char *const dictBasePath, const char *const contentFileNameSuffix) const {
|
bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const {
|
||||||
return DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
|
return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
|
||||||
contentFileNameSuffix, &mExpandableContentBuffer);
|
contentFileNameSuffix, &mExpandableContentBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,18 +18,18 @@
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
bool SparseTableDictContent::flush(const char *const dictBasePath,
|
bool SparseTableDictContent::flush(const char *const dictPath,
|
||||||
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
|
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
|
||||||
const char *const contentFileNameSuffix) const {
|
const char *const contentFileNameSuffix) const {
|
||||||
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, lookupTableFileNameSuffix,
|
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix,
|
||||||
&mExpandableLookupTableBuffer)){
|
&mExpandableLookupTableBuffer)){
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, addressTableFileNameSuffix,
|
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix,
|
||||||
&mExpandableAddressTableBuffer)) {
|
&mExpandableAddressTableBuffer)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, contentFileNameSuffix,
|
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix,
|
||||||
&mExpandableContentBuffer)) {
|
&mExpandableContentBuffer)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,15 +30,15 @@ namespace latinime {
|
||||||
// TODO: Support multiple contents.
|
// TODO: Support multiple contents.
|
||||||
class SparseTableDictContent : public DictContent {
|
class SparseTableDictContent : public DictContent {
|
||||||
public:
|
public:
|
||||||
AK_FORCE_INLINE SparseTableDictContent(const char *const dictDirPath,
|
AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath,
|
||||||
const char *const lookupTableFileName, const char *const addressTableFileName,
|
const char *const lookupTableFileName, const char *const addressTableFileName,
|
||||||
const char *const contentFileName, const bool isUpdatable,
|
const char *const contentFileName, const bool isUpdatable,
|
||||||
const int sparseTableBlockSize, const int sparseTableDataSize)
|
const int sparseTableBlockSize, const int sparseTableDataSize)
|
||||||
: mLookupTableBuffer(
|
: mLookupTableBuffer(
|
||||||
MmappedBuffer::openBuffer(dictDirPath, lookupTableFileName, isUpdatable)),
|
MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
|
||||||
mAddressTableBuffer(
|
mAddressTableBuffer(
|
||||||
MmappedBuffer::openBuffer(dictDirPath, addressTableFileName, isUpdatable)),
|
MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
|
||||||
mContentBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)),
|
mContentBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
||||||
mExpandableLookupTableBuffer(
|
mExpandableLookupTableBuffer(
|
||||||
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBuffer() : 0,
|
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBuffer() : 0,
|
||||||
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBufferSize() : 0,
|
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBufferSize() : 0,
|
||||||
|
|
|
@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition(
|
||||||
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
|
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) const {
|
bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const {
|
||||||
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
|
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
|
||||||
// table and write the new table to the file.
|
// table and write the new table to the file.
|
||||||
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
|
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
|
||||||
|
@ -63,12 +63,12 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) co
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return lookupTableToWrite.flush(dictBasePath,
|
return lookupTableToWrite.flush(dictPath,
|
||||||
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
||||||
} else {
|
} else {
|
||||||
// We can simply use this lookup table because the buffer size has not been
|
// We can simply use this lookup table because the buffer size has not been
|
||||||
// changed.
|
// changed.
|
||||||
return flush(dictBasePath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -28,8 +28,8 @@ class TerminalPositionLookupTable : public SingleDictContent {
|
||||||
public:
|
public:
|
||||||
typedef hash_map_compat<int, int> TerminalIdMap;
|
typedef hash_map_compat<int, int> TerminalIdMap;
|
||||||
|
|
||||||
TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable)
|
TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable)
|
||||||
: SingleDictContent(dictDirPath,
|
: SingleDictContent(dictPath,
|
||||||
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
|
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
|
||||||
mSize(getBuffer()->getTailPosition()
|
mSize(getBuffer()->getTailPosition()
|
||||||
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
|
/ Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {}
|
||||||
|
@ -44,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
|
||||||
return mSize;
|
return mSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool flushToFile(const char *const dictBasePath) const;
|
bool flushToFile(const char *const dictPath) const;
|
||||||
|
|
||||||
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);
|
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);
|
||||||
|
|
||||||
|
|
|
@ -27,10 +27,10 @@
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
|
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
|
||||||
const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) {
|
const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) {
|
||||||
const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false;
|
const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false;
|
||||||
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
|
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
|
||||||
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable));
|
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, headerBuffer, isUpdatable));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
||||||
|
@ -57,38 +57,38 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
||||||
const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */;
|
const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */;
|
||||||
char dictName[dictNameBufSize];
|
char dictName[dictNameBufSize];
|
||||||
FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName);
|
FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName);
|
||||||
const int dictBasePathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName);
|
const int dictPathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName);
|
||||||
char dictBasePath[dictBasePathBufSize];
|
char dictPath[dictPathBufSize];
|
||||||
FileUtils::getFilePath(tmpDirPath, dictName, dictBasePathBufSize, dictBasePath);
|
FileUtils::getFilePath(tmpDirPath, dictName, dictPathBufSize, dictPath);
|
||||||
|
|
||||||
// Write header file.
|
// Write header file.
|
||||||
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
|
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
|
||||||
Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) {
|
Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) {
|
||||||
AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath,
|
AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath,
|
||||||
Ver4DictConstants::HEADER_FILE_EXTENSION);
|
Ver4DictConstants::HEADER_FILE_EXTENSION);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// Write trie file.
|
// Write trie file.
|
||||||
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
|
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath,
|
||||||
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
|
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
|
||||||
AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
|
AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
|
||||||
Ver4DictConstants::TRIE_FILE_EXTENSION);
|
Ver4DictConstants::TRIE_FILE_EXTENSION);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
// Write dictionary contents.
|
// Write dictionary contents.
|
||||||
if (!mTerminalPositionLookupTable.flushToFile(dictBasePath)) {
|
if (!mTerminalPositionLookupTable.flushToFile(dictPath)) {
|
||||||
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
|
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mProbabilityDictContent.flushToFile(dictBasePath)) {
|
if (!mProbabilityDictContent.flushToFile(dictPath)) {
|
||||||
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
|
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mBigramDictContent.flushToFile(dictBasePath)) {
|
if (!mBigramDictContent.flushToFile(dictPath)) {
|
||||||
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
|
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mShortcutDictContent.flushToFile(dictBasePath)) {
|
if (!mShortcutDictContent.flushToFile(dictPath)) {
|
||||||
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
|
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -107,10 +107,10 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath,
|
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
|
||||||
const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable)
|
const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable)
|
||||||
: mHeaderBuffer(headerBuffer),
|
: mHeaderBuffer(headerBuffer),
|
||||||
mDictBuffer(MmappedBuffer::openBuffer(dictDirPath,
|
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
|
||||||
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
|
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
|
||||||
mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
|
mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
|
||||||
mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
|
mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(),
|
||||||
|
@ -118,12 +118,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath,
|
||||||
mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(),
|
mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(),
|
||||||
mDictBuffer.get()->getBufferSize(),
|
mDictBuffer.get()->getBufferSize(),
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mTerminalPositionLookupTable(dictDirPath, isUpdatable),
|
mTerminalPositionLookupTable(dictPath, isUpdatable),
|
||||||
mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
||||||
isUpdatable),
|
isUpdatable),
|
||||||
mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
||||||
isUpdatable),
|
isUpdatable),
|
||||||
mShortcutDictContent(dictDirPath, isUpdatable),
|
mShortcutDictContent(dictPath, isUpdatable),
|
||||||
mIsUpdatable(isUpdatable) {}
|
mIsUpdatable(isUpdatable) {}
|
||||||
|
|
||||||
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
|
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
|
||||||
|
|
|
@ -32,12 +32,9 @@
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath,
|
void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath,
|
||||||
const int unigramCount, const int bigramCount) const {
|
const int unigramCount, const int bigramCount) const {
|
||||||
const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
|
const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
|
||||||
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
|
|
||||||
char dirPath[dirPathBufSize];
|
|
||||||
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
|
|
||||||
BufferWithExtendableBuffer headerBuffer(
|
BufferWithExtendableBuffer headerBuffer(
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE);
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE);
|
||||||
const int extendedRegionSize = headerPolicy->getExtendedRegionSize()
|
const int extendedRegionSize = headerPolicy->getExtendedRegionSize()
|
||||||
|
@ -50,11 +47,11 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa
|
||||||
extendedRegionSize);
|
extendedRegionSize);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
|
mBuffers->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
|
void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
|
||||||
const char *const trieFilePath) {
|
const char *const dictDirPath) {
|
||||||
const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
|
const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy();
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
|
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
|
||||||
Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
|
Ver4DictBuffers::createVer4DictBuffers(headerPolicy));
|
||||||
|
@ -70,10 +67,7 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeAr
|
||||||
0 /* extendedRegionSize */)) {
|
0 /* extendedRegionSize */)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */;
|
dictBuffers.get()->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer);
|
||||||
char dirPath[dirPathBufSize];
|
|
||||||
FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath);
|
|
||||||
dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
|
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
|
||||||
|
|
|
@ -33,11 +33,10 @@ class Ver4PatriciaTrieWritingHelper {
|
||||||
Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers)
|
Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers)
|
||||||
: mBuffers(buffers) {}
|
: mBuffers(buffers) {}
|
||||||
|
|
||||||
void writeToDictFile(const char *const trieFilePath, const int unigramCount,
|
void writeToDictFile(const char *const dictDirPath, const int unigramCount,
|
||||||
const int bigramCount) const;
|
const int bigramCount) const;
|
||||||
|
|
||||||
void writeToDictFileWithGC(const int rootPtNodeArrayPos,
|
void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const dictDirPath);
|
||||||
const char *const trieFilePath);
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper);
|
||||||
|
|
|
@ -147,7 +147,7 @@ namespace latinime {
|
||||||
const char *const baseName = basename(filePathBuf);
|
const char *const baseName = basename(filePathBuf);
|
||||||
const int baseNameLength = strlen(baseName);
|
const int baseNameLength = strlen(baseName);
|
||||||
if (baseNameLength >= outNameBufSize) {
|
if (baseNameLength >= outNameBufSize) {
|
||||||
AKLOGE("outNameBufSize is too small. dirPath: %s, outNameBufSize: %d",
|
AKLOGE("outNameBufSize is too small. filePath: %s, outNameBufSize: %d",
|
||||||
filePath, outNameBufSize);
|
filePath, outNameBufSize);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,25 +39,15 @@ class ExclusiveOwnershipPointer {
|
||||||
deletePointersIfHavingOwnership();
|
deletePointersIfHavingOwnership();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Move the ownership.
|
|
||||||
AK_FORCE_INLINE ExclusiveOwnershipPointer<T> &operator=(
|
|
||||||
const ExclusiveOwnershipPointer<T> &pointer) {
|
|
||||||
// Delete pointers when this is an owner of another pointer.
|
|
||||||
deletePointersIfHavingOwnership();
|
|
||||||
mPointer = pointer.mPointer;
|
|
||||||
mSharedOwnerPtr = pointer.mSharedOwnerPtr;
|
|
||||||
transferOwnership(pointer);
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE T *get() const {
|
AK_FORCE_INLINE T *get() const {
|
||||||
return mPointer;
|
return mPointer;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// This class allows to copy and assign and ensures only one instance has the ownership of the
|
// This class allows to copy and ensures only one instance has the ownership of the
|
||||||
// managed pointer.
|
// managed pointer.
|
||||||
DISALLOW_DEFAULT_CONSTRUCTOR(ExclusiveOwnershipPointer);
|
DISALLOW_DEFAULT_CONSTRUCTOR(ExclusiveOwnershipPointer);
|
||||||
|
DISALLOW_ASSIGNMENT_OPERATOR(ExclusiveOwnershipPointer);
|
||||||
|
|
||||||
void transferOwnership(const ExclusiveOwnershipPointer<T> *const src) {
|
void transferOwnership(const ExclusiveOwnershipPointer<T> *const src) {
|
||||||
if (*mSharedOwnerPtr != src) {
|
if (*mSharedOwnerPtr != src) {
|
||||||
|
|
|
@ -114,7 +114,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
|
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
|
||||||
getContext().getCacheDir());
|
getContext().getCacheDir());
|
||||||
FileUtils.deleteRecursively(file);
|
FileUtils.deleteRecursively(file);
|
||||||
file.mkdir();
|
|
||||||
Map<String, String> attributeMap = new HashMap<String, String>();
|
Map<String, String> attributeMap = new HashMap<String, String>();
|
||||||
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
||||||
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
|
@ -125,7 +124,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
|
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
|
||||||
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
FormatSpec.VERSION4, attributeMap)) {
|
FormatSpec.VERSION4, attributeMap)) {
|
||||||
return new File(file, headerFileName);
|
return file;
|
||||||
} else {
|
} else {
|
||||||
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
|
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
|
||||||
+ headerFileName + " cannot be created.");
|
+ headerFileName + " cannot be created.");
|
||||||
|
|
|
@ -25,6 +25,7 @@ import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
|
||||||
import com.android.inputmethod.latin.makedict.CodePointUtils;
|
import com.android.inputmethod.latin.makedict.CodePointUtils;
|
||||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||||
|
import com.android.inputmethod.latin.utils.FileUtils;
|
||||||
import com.android.inputmethod.latin.utils.UnigramProperty;
|
import com.android.inputmethod.latin.utils.UnigramProperty;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -70,13 +71,12 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
Map<String, String> attributeMap = new HashMap<String, String>();
|
Map<String, String> attributeMap = new HashMap<String, String>();
|
||||||
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
||||||
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
|
|
||||||
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
FormatSpec.VERSION4, attributeMap)) {
|
FormatSpec.VERSION4, attributeMap)) {
|
||||||
return new File(file, headerFileName);
|
return file;
|
||||||
} else {
|
} else {
|
||||||
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
|
throw new IOException("Empty dictionary " + file.getAbsolutePath()
|
||||||
+ headerFileName + " cannot be created.");
|
+ " cannot be created.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -99,7 +99,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.close();
|
binaryDictionary.close();
|
||||||
assertFalse("binaryDictionary must be invalid after closing.",
|
assertFalse("binaryDictionary must be invalid after closing.",
|
||||||
binaryDictionary.isValidDictionary());
|
binaryDictionary.isValidDictionary());
|
||||||
dictFile.delete();
|
FileUtils.deleteRecursively(dictFile);
|
||||||
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */,
|
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */,
|
||||||
dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(),
|
dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(),
|
||||||
TEST_LOCALE, true /* isUpdatable */);
|
TEST_LOCALE, true /* isUpdatable */);
|
||||||
|
|
|
@ -20,6 +20,7 @@ import android.test.AndroidTestCase;
|
||||||
import android.test.suitebuilder.annotation.LargeTest;
|
import android.test.suitebuilder.annotation.LargeTest;
|
||||||
import android.util.Log;
|
import android.util.Log;
|
||||||
|
|
||||||
|
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
|
||||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
@ -136,7 +137,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||||
Log.d(TAG, "This test can be used for profiling.");
|
Log.d(TAG, "This test can be used for profiling.");
|
||||||
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
|
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
|
||||||
final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis();
|
final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis();
|
||||||
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix;
|
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
|
||||||
|
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
|
||||||
|
|
||||||
final int numberOfWords = 1000;
|
final int numberOfWords = 1000;
|
||||||
final Random random = new Random(123456);
|
final Random random = new Random(123456);
|
||||||
|
@ -151,7 +153,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||||
final File dictFile = new File(getContext().getFilesDir(), fileName);
|
final File dictFile = new File(getContext().getFilesDir(), fileName);
|
||||||
if (dictFile != null) {
|
if (dictFile != null) {
|
||||||
assertTrue(dictFile.exists());
|
assertTrue(dictFile.exists());
|
||||||
assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
|
|
||||||
dictFile.delete();
|
dictFile.delete();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -170,7 +171,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||||
// Create filename suffixes for this test.
|
// Create filename suffixes for this test.
|
||||||
for (int i = 0; i < numberOfLanguages; i++) {
|
for (int i = 0; i < numberOfLanguages; i++) {
|
||||||
testFilenameSuffixes[i] = "test_switching_languages" + i;
|
testFilenameSuffixes[i] = "test_switching_languages" + i;
|
||||||
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i];
|
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i]
|
||||||
|
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
|
||||||
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
|
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
|
||||||
clearHistory(testFilenameSuffixes[i]);
|
clearHistory(testFilenameSuffixes[i]);
|
||||||
}
|
}
|
||||||
|
@ -196,7 +198,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||||
for (final File file : dictFiles) {
|
for (final File file : dictFiles) {
|
||||||
if (file != null) {
|
if (file != null) {
|
||||||
assertTrue(file.exists());
|
assertTrue(file.exists());
|
||||||
assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
|
|
||||||
file.delete();
|
file.delete();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -214,11 +215,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||||
} finally {
|
} finally {
|
||||||
Log.d(TAG, "waiting for writing ...");
|
Log.d(TAG, "waiting for writing ...");
|
||||||
waitForWriting(testFilenameSuffix);
|
waitForWriting(testFilenameSuffix);
|
||||||
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix;
|
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
|
||||||
|
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
|
||||||
final File dictFile = new File(getContext().getFilesDir(), fileName);
|
final File dictFile = new File(getContext().getFilesDir(), fileName);
|
||||||
if (dictFile != null) {
|
if (dictFile != null) {
|
||||||
assertTrue(dictFile.exists());
|
assertTrue(dictFile.exists());
|
||||||
assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
|
|
||||||
dictFile.delete();
|
dictFile.delete();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue