parent
3339224813
commit
4ce480d5ce
|
@ -46,15 +46,16 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
|
||||||
char sourceDirChars[sourceDirUtf8Length + 1];
|
char sourceDirChars[sourceDirUtf8Length + 1];
|
||||||
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
|
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
|
||||||
sourceDirChars[sourceDirUtf8Length] = '\0';
|
sourceDirChars[sourceDirUtf8Length] = '\0';
|
||||||
DictionaryStructureWithBufferPolicy::StructurePolicyPtr dictionaryStructureWithBufferPolicy =
|
DictionaryStructureWithBufferPolicy::StructurePolicyPtr dictionaryStructureWithBufferPolicy(
|
||||||
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
|
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
|
||||||
sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
|
sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
|
||||||
isUpdatable == JNI_TRUE);
|
isUpdatable == JNI_TRUE));
|
||||||
if (!dictionaryStructureWithBufferPolicy.get()) {
|
if (!dictionaryStructureWithBufferPolicy) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
|
Dictionary *const dictionary =
|
||||||
|
new Dictionary(env, std::move(dictionaryStructureWithBufferPolicy));
|
||||||
PROF_END(66);
|
PROF_END(66);
|
||||||
PROF_CLOSE;
|
PROF_CLOSE;
|
||||||
return reinterpret_cast<jlong>(dictionary);
|
return reinterpret_cast<jlong>(dictionary);
|
||||||
|
|
|
@ -34,9 +34,9 @@ namespace latinime {
|
||||||
|
|
||||||
const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32;
|
const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32;
|
||||||
|
|
||||||
Dictionary::Dictionary(JNIEnv *env, const DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
Dictionary::Dictionary(JNIEnv *env, DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
&dictionaryStructureWithBufferPolicy)
|
dictionaryStructureWithBufferPolicy)
|
||||||
: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
|
: mDictionaryStructureWithBufferPolicy(std::move(dictionaryStructureWithBufferPolicy)),
|
||||||
mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy.get())),
|
mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy.get())),
|
||||||
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
|
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
|
||||||
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
|
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
|
||||||
|
@ -53,7 +53,7 @@ int Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession
|
||||||
if (suggestOptions->isGesture()) {
|
if (suggestOptions->isGesture()) {
|
||||||
DicTraverseSession::initSessionInstance(
|
DicTraverseSession::initSessionInstance(
|
||||||
traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
|
traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
|
||||||
result = mGestureSuggest.get()->getSuggestions(proximityInfo, traverseSession, xcoordinates,
|
result = mGestureSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
|
||||||
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, outWords,
|
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, outWords,
|
||||||
outputScores, spaceIndices, outputTypes, outputAutoCommitFirstWordConfidence);
|
outputScores, spaceIndices, outputTypes, outputAutoCommitFirstWordConfidence);
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
|
@ -63,7 +63,7 @@ int Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession
|
||||||
} else {
|
} else {
|
||||||
DicTraverseSession::initSessionInstance(
|
DicTraverseSession::initSessionInstance(
|
||||||
traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
|
traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions);
|
||||||
result = mTypingSuggest.get()->getSuggestions(proximityInfo, traverseSession, xcoordinates,
|
result = mTypingSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
|
||||||
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint,
|
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint,
|
||||||
outWords, outputScores, spaceIndices, outputTypes,
|
outWords, outputScores, spaceIndices, outputTypes,
|
||||||
outputAutoCommitFirstWordConfidence);
|
outputAutoCommitFirstWordConfidence);
|
||||||
|
@ -78,7 +78,7 @@ int Dictionary::getBigrams(const int *word, int length, int *outWords, int *outp
|
||||||
int *outputTypes) const {
|
int *outputTypes) const {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
if (length <= 0) return 0;
|
if (length <= 0) return 0;
|
||||||
return mBigramDictionary.get()->getPredictions(word, length, outWords, outputScores,
|
return mBigramDictionary->getPredictions(word, length, outWords, outputScores,
|
||||||
outputTypes);
|
outputTypes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -95,7 +95,7 @@ int Dictionary::getProbability(const int *word, int length) const {
|
||||||
int Dictionary::getBigramProbability(const int *word0, int length0, const int *word1,
|
int Dictionary::getBigramProbability(const int *word0, int length0, const int *word1,
|
||||||
int length1) const {
|
int length1) const {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
return mBigramDictionary.get()->getBigramProbability(word0, length0, word1, length1);
|
return mBigramDictionary->getBigramProbability(word0, length0, word1, length1);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability,
|
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability,
|
||||||
|
@ -103,7 +103,7 @@ void Dictionary::addUnigramWord(const int *const word, const int length, const i
|
||||||
const int shortcutProbability, const bool isNotAWord, const bool isBlacklisted,
|
const int shortcutProbability, const bool isNotAWord, const bool isBlacklisted,
|
||||||
const int timestamp) {
|
const int timestamp) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy.get()->addUnigramWord(word, length, probability,
|
mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability,
|
||||||
shortcutTargetCodePoints, shortcutLength, shortcutProbability, isNotAWord,
|
shortcutTargetCodePoints, shortcutLength, shortcutProbability, isNotAWord,
|
||||||
isBlacklisted, timestamp);
|
isBlacklisted, timestamp);
|
||||||
}
|
}
|
||||||
|
@ -111,48 +111,48 @@ void Dictionary::addUnigramWord(const int *const word, const int length, const i
|
||||||
void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
|
void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
|
||||||
const int length1, const int probability, const int timestamp) {
|
const int length1, const int probability, const int timestamp) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy.get()->addBigramWords(word0, length0, word1, length1,
|
mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
|
||||||
probability, timestamp);
|
probability, timestamp);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::removeBigramWords(const int *const word0, const int length0,
|
void Dictionary::removeBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1) {
|
const int *const word1, const int length1) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy.get()->removeBigramWords(word0, length0, word1, length1);
|
mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::flush(const char *const filePath) {
|
void Dictionary::flush(const char *const filePath) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy.get()->flush(filePath);
|
mDictionaryStructureWithBufferPolicy->flush(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::flushWithGC(const char *const filePath) {
|
void Dictionary::flushWithGC(const char *const filePath) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy.get()->flushWithGC(filePath);
|
mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
|
bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
return mDictionaryStructureWithBufferPolicy.get()->needsToRunGC(mindsBlockByGC);
|
return mDictionaryStructureWithBufferPolicy->needsToRunGC(mindsBlockByGC);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::getProperty(const char *const query, const int queryLength, char *const outResult,
|
void Dictionary::getProperty(const char *const query, const int queryLength, char *const outResult,
|
||||||
const int maxResultLength) {
|
const int maxResultLength) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
return mDictionaryStructureWithBufferPolicy.get()->getProperty(query, queryLength, outResult,
|
return mDictionaryStructureWithBufferPolicy->getProperty(query, queryLength, outResult,
|
||||||
maxResultLength);
|
maxResultLength);
|
||||||
}
|
}
|
||||||
|
|
||||||
const WordProperty Dictionary::getWordProperty(const int *const codePoints,
|
const WordProperty Dictionary::getWordProperty(const int *const codePoints,
|
||||||
const int codePointCount) {
|
const int codePointCount) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
return mDictionaryStructureWithBufferPolicy.get()->getWordProperty(
|
return mDictionaryStructureWithBufferPolicy->getWordProperty(
|
||||||
codePoints, codePointCount);
|
codePoints, codePointCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
int Dictionary::getNextWordAndNextToken(const int token, int *const outCodePoints) {
|
int Dictionary::getNextWordAndNextToken(const int token, int *const outCodePoints) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
return mDictionaryStructureWithBufferPolicy.get()->getNextWordAndNextToken(
|
return mDictionaryStructureWithBufferPolicy->getNextWordAndNextToken(
|
||||||
token, outCodePoints);
|
token, outCodePoints);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
#define LATINIME_DICTIONARY_H
|
#define LATINIME_DICTIONARY_H
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "jni.h"
|
#include "jni.h"
|
||||||
|
@ -26,7 +27,6 @@
|
||||||
#include "suggest/core/policy/dictionary_header_structure_policy.h"
|
#include "suggest/core/policy/dictionary_header_structure_policy.h"
|
||||||
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
||||||
#include "suggest/core/suggest_interface.h"
|
#include "suggest/core/suggest_interface.h"
|
||||||
#include "utils/exclusive_ownership_pointer.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -58,8 +58,8 @@ class Dictionary {
|
||||||
static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
|
static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
|
||||||
static const int KIND_FLAG_EXACT_MATCH = 0x40000000;
|
static const int KIND_FLAG_EXACT_MATCH = 0x40000000;
|
||||||
|
|
||||||
Dictionary(JNIEnv *env, const DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
Dictionary(JNIEnv *env, DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
&dictionaryStructureWithBufferPolicy);
|
dictionaryStructureWithBufferPolicy);
|
||||||
|
|
||||||
int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
|
int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
|
||||||
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
|
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
|
||||||
|
@ -108,8 +108,8 @@ class Dictionary {
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
|
||||||
|
|
||||||
typedef ExclusiveOwnershipPointer<BigramDictionary> BigramDictionaryPtr;
|
typedef std::unique_ptr<BigramDictionary> BigramDictionaryPtr;
|
||||||
typedef ExclusiveOwnershipPointer<SuggestInterface> SuggestInterfacePtr;
|
typedef std::unique_ptr<SuggestInterface> SuggestInterfacePtr;
|
||||||
|
|
||||||
static const int HEADER_ATTRIBUTE_BUFFER_SIZE;
|
static const int HEADER_ATTRIBUTE_BUFFER_SIZE;
|
||||||
|
|
||||||
|
|
|
@ -17,9 +17,10 @@
|
||||||
#ifndef LATINIME_DICTIONARY_STRUCTURE_POLICY_H
|
#ifndef LATINIME_DICTIONARY_STRUCTURE_POLICY_H
|
||||||
#define LATINIME_DICTIONARY_STRUCTURE_POLICY_H
|
#define LATINIME_DICTIONARY_STRUCTURE_POLICY_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/dictionary/word_property.h"
|
#include "suggest/core/dictionary/word_property.h"
|
||||||
#include "utils/exclusive_ownership_pointer.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -35,7 +36,7 @@ class DictionaryShortcutsStructurePolicy;
|
||||||
*/
|
*/
|
||||||
class DictionaryStructureWithBufferPolicy {
|
class DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
typedef ExclusiveOwnershipPointer<DictionaryStructureWithBufferPolicy> StructurePolicyPtr;
|
typedef std::unique_ptr<DictionaryStructureWithBufferPolicy> StructurePolicyPtr;
|
||||||
|
|
||||||
virtual ~DictionaryStructureWithBufferPolicy() {}
|
virtual ~DictionaryStructureWithBufferPolicy() {}
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,7 @@ namespace latinime {
|
||||||
if (isUpdatable) {
|
if (isUpdatable) {
|
||||||
AKLOGE("One file dictionaries don't support updating. path: %s", path);
|
AKLOGE("One file dictionaries don't support updating. path: %s", path);
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
return newPolicyforFileDict(path, bufOffset, size);
|
return newPolicyforFileDict(path, bufOffset, size);
|
||||||
}
|
}
|
||||||
|
@ -55,13 +55,13 @@ namespace latinime {
|
||||||
getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath);
|
getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath);
|
||||||
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
||||||
// MmappedBufferPtr if the instance has the responsibility.
|
// MmappedBufferPtr if the instance has the responsibility.
|
||||||
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(headerFilePath,
|
MmappedBuffer::MmappedBufferPtr mmappedBuffer(
|
||||||
isUpdatable);
|
MmappedBuffer::openBuffer(headerFilePath, isUpdatable));
|
||||||
if (!mmappedBuffer.get()) {
|
if (!mmappedBuffer) {
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(),
|
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
|
||||||
mmappedBuffer.get()->getBufferSize())) {
|
mmappedBuffer->getBufferSize())) {
|
||||||
case FormatUtils::VERSION_2:
|
case FormatUtils::VERSION_2:
|
||||||
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
|
AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
|
||||||
break;
|
break;
|
||||||
|
@ -72,25 +72,25 @@ namespace latinime {
|
||||||
Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
|
Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) {
|
||||||
AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
|
AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path);
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
|
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
|
||||||
Ver4DictBuffers::openVer4DictBuffers(dictPath, mmappedBuffer);
|
Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer)));
|
||||||
if (!dictBuffers.get() || !dictBuffers.get()->isValid()) {
|
if (!dictBuffers || !dictBuffers->isValid()) {
|
||||||
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
|
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
|
||||||
path);
|
path);
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
||||||
new Ver4PatriciaTriePolicy(dictBuffers));
|
new Ver4PatriciaTriePolicy(std::move(dictBuffers)));
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
|
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
|
||||||
|
@ -98,16 +98,16 @@ namespace latinime {
|
||||||
const char *const path, const int bufOffset, const int size) {
|
const char *const path, const int bufOffset, const int size) {
|
||||||
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
||||||
// MmappedBufferPtr if the instance has the responsibility.
|
// MmappedBufferPtr if the instance has the responsibility.
|
||||||
MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset,
|
MmappedBuffer::MmappedBufferPtr mmappedBuffer(
|
||||||
size, false /* isUpdatable */);
|
MmappedBuffer::openBuffer(path, bufOffset, size, false /* isUpdatable */));
|
||||||
if (!mmappedBuffer.get()) {
|
if (!mmappedBuffer) {
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(),
|
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(),
|
||||||
mmappedBuffer.get()->getBufferSize())) {
|
mmappedBuffer->getBufferSize())) {
|
||||||
case FormatUtils::VERSION_2:
|
case FormatUtils::VERSION_2:
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
||||||
new PatriciaTriePolicy(mmappedBuffer));
|
new PatriciaTriePolicy(std::move(mmappedBuffer)));
|
||||||
case FormatUtils::VERSION_4:
|
case FormatUtils::VERSION_4:
|
||||||
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
|
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
|
||||||
break;
|
break;
|
||||||
|
@ -116,7 +116,7 @@ namespace latinime {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0);
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir(
|
/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir(
|
||||||
|
|
|
@ -21,7 +21,6 @@
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
||||||
#include "utils/exclusive_ownership_pointer.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
|
|
@ -37,12 +37,11 @@ class DicNodeVector;
|
||||||
|
|
||||||
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
PatriciaTriePolicy(const MmappedBuffer::MmappedBufferPtr &mmappedBuffer)
|
PatriciaTriePolicy(MmappedBuffer::MmappedBufferPtr mmappedBuffer)
|
||||||
: mMmappedBuffer(mmappedBuffer),
|
: mMmappedBuffer(std::move(mmappedBuffer)),
|
||||||
mHeaderPolicy(mMmappedBuffer.get()->getBuffer(), FormatUtils::VERSION_2),
|
mHeaderPolicy(mMmappedBuffer->getBuffer(), FormatUtils::VERSION_2),
|
||||||
mDictRoot(mMmappedBuffer.get()->getBuffer() + mHeaderPolicy.getSize()),
|
mDictRoot(mMmappedBuffer->getBuffer() + mHeaderPolicy.getSize()),
|
||||||
mDictBufferSize(mMmappedBuffer.get()->getBufferSize()
|
mDictBufferSize(mMmappedBuffer->getBufferSize() - mHeaderPolicy.getSize()),
|
||||||
- mHeaderPolicy.getSize()),
|
|
||||||
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot),
|
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot),
|
||||||
mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy),
|
mPtNodeReader(mDictRoot, mDictBufferSize, &mBigramListPolicy, &mShortcutListPolicy),
|
||||||
mPtNodeArrayReader(mDictRoot, mDictBufferSize),
|
mPtNodeArrayReader(mDictRoot, mDictBufferSize),
|
||||||
|
|
|
@ -31,15 +31,14 @@ class SingleDictContent : public DictContent {
|
||||||
SingleDictContent(const char *const dictPath, const char *const contentFileName,
|
SingleDictContent(const char *const dictPath, const char *const contentFileName,
|
||||||
const bool isUpdatable)
|
const bool isUpdatable)
|
||||||
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
: mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
||||||
mExpandableContentBuffer(mMmappedBuffer.get() ?
|
mExpandableContentBuffer(mMmappedBuffer ? mMmappedBuffer->getBuffer() : nullptr,
|
||||||
mMmappedBuffer.get()->getBuffer() : nullptr,
|
mMmappedBuffer ? mMmappedBuffer->getBufferSize() : 0,
|
||||||
mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0,
|
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mIsValid(mMmappedBuffer.get()) {}
|
mIsValid(mMmappedBuffer) {}
|
||||||
|
|
||||||
SingleDictContent()
|
SingleDictContent()
|
||||||
: mMmappedBuffer(0), mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
: mMmappedBuffer(nullptr),
|
||||||
mIsValid(true) {}
|
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mIsValid(true) {}
|
||||||
|
|
||||||
virtual ~SingleDictContent() {}
|
virtual ~SingleDictContent() {}
|
||||||
|
|
||||||
|
|
|
@ -38,26 +38,25 @@ class SparseTableDictContent : public DictContent {
|
||||||
MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
|
MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)),
|
||||||
mAddressTableBuffer(
|
mAddressTableBuffer(
|
||||||
MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
|
MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)),
|
||||||
mContentBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
mContentBuffer(
|
||||||
|
MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)),
|
||||||
mExpandableLookupTableBuffer(
|
mExpandableLookupTableBuffer(
|
||||||
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBuffer() : nullptr,
|
mLookupTableBuffer ? mLookupTableBuffer->getBuffer() : nullptr,
|
||||||
mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBufferSize() : 0,
|
mLookupTableBuffer ? mLookupTableBuffer->getBufferSize() : 0,
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mExpandableAddressTableBuffer(
|
mExpandableAddressTableBuffer(
|
||||||
mAddressTableBuffer.get() ? mAddressTableBuffer.get()->getBuffer() : nullptr,
|
mAddressTableBuffer ? mAddressTableBuffer->getBuffer() : nullptr,
|
||||||
mAddressTableBuffer.get() ? mAddressTableBuffer.get()->getBufferSize() : 0,
|
mAddressTableBuffer ? mAddressTableBuffer->getBufferSize() : 0,
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mExpandableContentBuffer(mContentBuffer.get() ?
|
mExpandableContentBuffer(mContentBuffer ? mContentBuffer->getBuffer() : nullptr,
|
||||||
mContentBuffer.get()->getBuffer() : nullptr,
|
mContentBuffer ? mContentBuffer->getBufferSize() : 0,
|
||||||
mContentBuffer.get() ? mContentBuffer.get()->getBufferSize() : 0,
|
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
|
mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
|
||||||
sparseTableBlockSize, sparseTableDataSize),
|
sparseTableBlockSize, sparseTableDataSize),
|
||||||
mIsValid(mLookupTableBuffer.get() && mAddressTableBuffer.get()
|
mIsValid(mLookupTableBuffer && mAddressTableBuffer && mContentBuffer) {}
|
||||||
&& mContentBuffer.get()) {}
|
|
||||||
|
|
||||||
SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
|
SparseTableDictContent(const int sparseTableBlockSize, const int sparseTableDataSize)
|
||||||
: mLookupTableBuffer(0), mAddressTableBuffer(0), mContentBuffer(0),
|
: mLookupTableBuffer(), mAddressTableBuffer(), mContentBuffer(),
|
||||||
mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
mExpandableLookupTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||||
mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
mExpandableAddressTableBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||||
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
mExpandableContentBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||||
|
|
|
@ -27,15 +27,15 @@
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
|
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
|
||||||
const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) {
|
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer) {
|
||||||
if (!headerBuffer.get()) {
|
if (!headerBuffer) {
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
AKLOGE("The header buffer must be valid to open ver4 dict buffers.");
|
AKLOGE("The header buffer must be valid to open ver4 dict buffers.");
|
||||||
return Ver4DictBuffersPtr(nullptr);
|
return Ver4DictBuffersPtr(nullptr);
|
||||||
}
|
}
|
||||||
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
|
// TODO: take only dictDirPath, and open both header and trie files in the constructor below
|
||||||
return Ver4DictBuffersPtr(new Ver4DictBuffers(
|
const bool isUpdatable = headerBuffer->isUpdatable();
|
||||||
dictPath, headerBuffer, headerBuffer.get()->isUpdatable()));
|
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
||||||
|
@ -113,27 +113,25 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
|
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
|
||||||
const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable)
|
MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable)
|
||||||
: mHeaderBuffer(headerBuffer),
|
: mHeaderBuffer(std::move(headerBuffer)),
|
||||||
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
|
mDictBuffer(MmappedBuffer::openBuffer(dictPath,
|
||||||
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
|
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
|
||||||
mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4),
|
mHeaderPolicy(mHeaderBuffer->getBuffer(), FormatUtils::VERSION_4),
|
||||||
mExpandableHeaderBuffer(headerBuffer.get() ? headerBuffer.get()->getBuffer() : nullptr,
|
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
|
||||||
mHeaderPolicy.getSize(),
|
mHeaderPolicy.getSize(),
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mExpandableTrieBuffer(mDictBuffer.get() ? mDictBuffer.get()->getBuffer() : nullptr,
|
mExpandableTrieBuffer(mDictBuffer ? mDictBuffer->getBuffer() : nullptr,
|
||||||
mDictBuffer.get() ? mDictBuffer.get()->getBufferSize() : 0,
|
mDictBuffer ? mDictBuffer->getBufferSize() : 0,
|
||||||
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
|
||||||
mTerminalPositionLookupTable(dictPath, isUpdatable),
|
mTerminalPositionLookupTable(dictPath, isUpdatable),
|
||||||
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
|
||||||
isUpdatable),
|
mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable),
|
||||||
mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(),
|
|
||||||
isUpdatable),
|
|
||||||
mShortcutDictContent(dictPath, isUpdatable),
|
mShortcutDictContent(dictPath, isUpdatable),
|
||||||
mIsUpdatable(isUpdatable) {}
|
mIsUpdatable(isUpdatable) {}
|
||||||
|
|
||||||
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
|
Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy)
|
||||||
: mHeaderBuffer(0), mDictBuffer(0), mHeaderPolicy(),
|
: mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(),
|
||||||
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||||
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
|
||||||
mTerminalPositionLookupTable(),
|
mTerminalPositionLookupTable(),
|
||||||
|
|
|
@ -17,6 +17,8 @@
|
||||||
#ifndef LATINIME_VER4_DICT_BUFFER_H
|
#ifndef LATINIME_VER4_DICT_BUFFER_H
|
||||||
#define LATINIME_VER4_DICT_BUFFER_H
|
#define LATINIME_VER4_DICT_BUFFER_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
|
#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
|
||||||
|
@ -31,10 +33,10 @@ namespace latinime {
|
||||||
|
|
||||||
class Ver4DictBuffers {
|
class Ver4DictBuffers {
|
||||||
public:
|
public:
|
||||||
typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr;
|
typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr;
|
||||||
|
|
||||||
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
|
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
|
||||||
const MmappedBuffer::MmappedBufferPtr &headerBuffer);
|
MmappedBuffer::MmappedBufferPtr headerBuffer);
|
||||||
|
|
||||||
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
|
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
|
||||||
const HeaderPolicy *const headerPolicy) {
|
const HeaderPolicy *const headerPolicy) {
|
||||||
|
@ -42,7 +44,7 @@ class Ver4DictBuffers {
|
||||||
}
|
}
|
||||||
|
|
||||||
AK_FORCE_INLINE bool isValid() const {
|
AK_FORCE_INLINE bool isValid() const {
|
||||||
return mHeaderBuffer.get() && mDictBuffer.get() && mHeaderPolicy.isValid()
|
return mHeaderBuffer && mDictBuffer && mHeaderPolicy.isValid()
|
||||||
&& mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid()
|
&& mProbabilityDictContent.isValid() && mTerminalPositionLookupTable.isValid()
|
||||||
&& mBigramDictContent.isValid() && mShortcutDictContent.isValid();
|
&& mBigramDictContent.isValid() && mShortcutDictContent.isValid();
|
||||||
}
|
}
|
||||||
|
@ -118,7 +120,7 @@ class Ver4DictBuffers {
|
||||||
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
|
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
|
||||||
|
|
||||||
Ver4DictBuffers(const char *const dictDirPath,
|
Ver4DictBuffers(const char *const dictDirPath,
|
||||||
const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable);
|
const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable);
|
||||||
|
|
||||||
Ver4DictBuffers(const HeaderPolicy *const headerPolicy);
|
Ver4DictBuffers(const HeaderPolicy *const headerPolicy);
|
||||||
|
|
||||||
|
|
|
@ -135,7 +135,7 @@ int Ver4PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) con
|
||||||
if (ptNodeParams.isDeleted()) {
|
if (ptNodeParams.isDeleted()) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
return mBuffers.get()->getShortcutDictContent()->getShortcutListHeadPos(
|
return mBuffers->getShortcutDictContent()->getShortcutListHeadPos(
|
||||||
ptNodeParams.getTerminalId());
|
ptNodeParams.getTerminalId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -147,7 +147,7 @@ int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) cons
|
||||||
if (ptNodeParams.isDeleted()) {
|
if (ptNodeParams.isDeleted()) {
|
||||||
return NOT_A_DICT_POS;
|
return NOT_A_DICT_POS;
|
||||||
}
|
}
|
||||||
return mBuffers.get()->getBigramDictContent()->getBigramListHeadPos(
|
return mBuffers->getBigramDictContent()->getBigramListHeadPos(
|
||||||
ptNodeParams.getTerminalId());
|
ptNodeParams.getTerminalId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -155,7 +155,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
|
||||||
const int probability, const int *const shortcutTargetCodePoints, const int shortcutLength,
|
const int probability, const int *const shortcutTargetCodePoints, const int shortcutLength,
|
||||||
const int shortcutProbability, const bool isNotAWord, const bool isBlacklisted,
|
const int shortcutProbability, const bool isNotAWord, const bool isBlacklisted,
|
||||||
const int timestamp) {
|
const int timestamp) {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
|
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -205,7 +205,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len
|
||||||
bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
|
bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1, const int probability,
|
const int *const word1, const int length1, const int probability,
|
||||||
const int timestamp) {
|
const int timestamp) {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -243,7 +243,7 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le
|
||||||
|
|
||||||
bool Ver4PatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
|
bool Ver4PatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1) {
|
const int *const word1, const int length1) {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -276,7 +276,7 @@ bool Ver4PatriciaTriePolicy::removeBigramWords(const int *const word0, const int
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -287,7 +287,7 @@ void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -298,11 +298,11 @@ void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
||||||
if (!mBuffers.get()->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
|
AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (mBuffers.get()->isNearSizeLimit()) {
|
if (mBuffers->isNearSizeLimit()) {
|
||||||
// Additional buffer size is near the limit.
|
// Additional buffer size is near the limit.
|
||||||
return true;
|
return true;
|
||||||
} else if (mHeaderPolicy->getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
|
} else if (mHeaderPolicy->getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize()
|
||||||
|
@ -354,7 +354,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
|
||||||
std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
|
std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
|
||||||
ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
|
ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
|
||||||
const ProbabilityEntry probabilityEntry =
|
const ProbabilityEntry probabilityEntry =
|
||||||
mBuffers.get()->getProbabilityDictContent()->getProbabilityEntry(
|
mBuffers->getProbabilityDictContent()->getProbabilityEntry(
|
||||||
ptNodeParams.getTerminalId());
|
ptNodeParams.getTerminalId());
|
||||||
const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();
|
const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();
|
||||||
// Fetch bigram information.
|
// Fetch bigram information.
|
||||||
|
@ -362,9 +362,9 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
|
||||||
const int bigramListPos = getBigramsPositionOfPtNode(ptNodePos);
|
const int bigramListPos = getBigramsPositionOfPtNode(ptNodePos);
|
||||||
if (bigramListPos != NOT_A_DICT_POS) {
|
if (bigramListPos != NOT_A_DICT_POS) {
|
||||||
int bigramWord1CodePoints[MAX_WORD_LENGTH];
|
int bigramWord1CodePoints[MAX_WORD_LENGTH];
|
||||||
const BigramDictContent *const bigramDictContent = mBuffers.get()->getBigramDictContent();
|
const BigramDictContent *const bigramDictContent = mBuffers->getBigramDictContent();
|
||||||
const TerminalPositionLookupTable *const terminalPositionLookupTable =
|
const TerminalPositionLookupTable *const terminalPositionLookupTable =
|
||||||
mBuffers.get()->getTerminalPositionLookupTable();
|
mBuffers->getTerminalPositionLookupTable();
|
||||||
bool hasNext = true;
|
bool hasNext = true;
|
||||||
int readingPos = bigramListPos;
|
int readingPos = bigramListPos;
|
||||||
while (hasNext) {
|
while (hasNext) {
|
||||||
|
@ -400,7 +400,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
|
||||||
if (shortcutPos != NOT_A_DICT_POS) {
|
if (shortcutPos != NOT_A_DICT_POS) {
|
||||||
int shortcutTarget[MAX_WORD_LENGTH];
|
int shortcutTarget[MAX_WORD_LENGTH];
|
||||||
const ShortcutDictContent *const shortcutDictContent =
|
const ShortcutDictContent *const shortcutDictContent =
|
||||||
mBuffers.get()->getShortcutDictContent();
|
mBuffers->getShortcutDictContent();
|
||||||
bool hasNext = true;
|
bool hasNext = true;
|
||||||
while (hasNext) {
|
while (hasNext) {
|
||||||
int shortcutTargetLength = 0;
|
int shortcutTargetLength = 0;
|
||||||
|
|
|
@ -37,17 +37,16 @@ namespace latinime {
|
||||||
class DicNode;
|
class DicNode;
|
||||||
class DicNodeVector;
|
class DicNodeVector;
|
||||||
|
|
||||||
// TODO: Implement.
|
|
||||||
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers)
|
Ver4PatriciaTriePolicy(Ver4DictBuffers::Ver4DictBuffersPtr buffers)
|
||||||
: mBuffers(buffers), mHeaderPolicy(mBuffers.get()->getHeaderPolicy()),
|
: mBuffers(std::move(buffers)), mHeaderPolicy(mBuffers->getHeaderPolicy()),
|
||||||
mDictBuffer(mBuffers.get()->getWritableTrieBuffer()),
|
mDictBuffer(mBuffers->getWritableTrieBuffer()),
|
||||||
mBigramPolicy(mBuffers.get()->getMutableBigramDictContent(),
|
mBigramPolicy(mBuffers->getMutableBigramDictContent(),
|
||||||
mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy),
|
mBuffers->getTerminalPositionLookupTable(), mHeaderPolicy),
|
||||||
mShortcutPolicy(mBuffers.get()->getMutableShortcutDictContent(),
|
mShortcutPolicy(mBuffers->getMutableShortcutDictContent(),
|
||||||
mBuffers.get()->getTerminalPositionLookupTable()),
|
mBuffers->getTerminalPositionLookupTable()),
|
||||||
mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent(), mHeaderPolicy),
|
mNodeReader(mDictBuffer, mBuffers->getProbabilityDictContent(), mHeaderPolicy),
|
||||||
mPtNodeArrayReader(mDictBuffer),
|
mPtNodeArrayReader(mDictBuffer),
|
||||||
mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader,
|
mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader,
|
||||||
&mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy),
|
&mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy),
|
||||||
|
@ -132,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
|
static const int MARGIN_TO_REFUSE_DYNAMIC_OPERATIONS;
|
||||||
static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
|
static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS;
|
||||||
|
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
|
const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
|
||||||
const HeaderPolicy *const mHeaderPolicy;
|
const HeaderPolicy *const mHeaderPolicy;
|
||||||
BufferWithExtendableBuffer *const mDictBuffer;
|
BufferWithExtendableBuffer *const mDictBuffer;
|
||||||
Ver4BigramListPolicy mBigramPolicy;
|
Ver4BigramListPolicy mBigramPolicy;
|
||||||
|
|
|
@ -67,7 +67,7 @@ bool Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeAr
|
||||||
unigramCount, bigramCount, 0 /* extendedRegionSize */, &headerBuffer)) {
|
unigramCount, bigramCount, 0 /* extendedRegionSize */, &headerBuffer)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return dictBuffers.get()->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer);
|
return dictBuffers->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
|
bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
|
||||||
|
|
|
@ -48,17 +48,17 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
|
||||||
const std::vector<int> localeAsCodePointVector,
|
const std::vector<int> localeAsCodePointVector,
|
||||||
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
|
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
|
||||||
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
|
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
|
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
|
||||||
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
|
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy));
|
||||||
headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,
|
headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,
|
||||||
0 /* unigramCount */, 0 /* bigramCount */,
|
0 /* unigramCount */, 0 /* bigramCount */,
|
||||||
0 /* extendedRegionSize */, dictBuffers.get()->getWritableHeaderBuffer());
|
0 /* extendedRegionSize */, dictBuffers->getWritableHeaderBuffer());
|
||||||
if (!DynamicPtWritingUtils::writeEmptyDictionary(
|
if (!DynamicPtWritingUtils::writeEmptyDictionary(
|
||||||
dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) {
|
dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) {
|
||||||
AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
|
AKLOGE("Empty ver4 dictionary structure cannot be created on memory.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return dictBuffers.get()->flush(dirPath);
|
return dictBuffers->flush(dirPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
|
/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
|
||||||
|
|
|
@ -33,7 +33,7 @@ namespace latinime {
|
||||||
const int mmapFd = open(path, O_RDONLY);
|
const int mmapFd = open(path, O_RDONLY);
|
||||||
if (mmapFd < 0) {
|
if (mmapFd < 0) {
|
||||||
AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno);
|
AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno);
|
||||||
return MmappedBufferPtr(0);
|
return MmappedBufferPtr(nullptr);
|
||||||
}
|
}
|
||||||
const int pagesize = sysconf(_SC_PAGESIZE);
|
const int pagesize = sysconf(_SC_PAGESIZE);
|
||||||
const int offset = bufferOffset % pagesize;
|
const int offset = bufferOffset % pagesize;
|
||||||
|
@ -45,13 +45,13 @@ namespace latinime {
|
||||||
if (mmappedBuffer == MAP_FAILED) {
|
if (mmappedBuffer == MAP_FAILED) {
|
||||||
AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
|
AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
|
||||||
close(mmapFd);
|
close(mmapFd);
|
||||||
return MmappedBufferPtr(0);
|
return MmappedBufferPtr(nullptr);
|
||||||
}
|
}
|
||||||
uint8_t *const buffer = static_cast<uint8_t *>(mmappedBuffer) + offset;
|
uint8_t *const buffer = static_cast<uint8_t *>(mmappedBuffer) + offset;
|
||||||
if (!buffer) {
|
if (!buffer) {
|
||||||
AKLOGE("DICT: buffer is null");
|
AKLOGE("DICT: buffer is null");
|
||||||
close(mmapFd);
|
close(mmapFd);
|
||||||
return MmappedBufferPtr(0);
|
return MmappedBufferPtr(nullptr);
|
||||||
}
|
}
|
||||||
return MmappedBufferPtr(new MmappedBuffer(buffer, bufferSize, mmappedBuffer, alignedSize,
|
return MmappedBufferPtr(new MmappedBuffer(buffer, bufferSize, mmappedBuffer, alignedSize,
|
||||||
mmapFd, isUpdatable));
|
mmapFd, isUpdatable));
|
||||||
|
@ -61,7 +61,7 @@ namespace latinime {
|
||||||
const char *const path, const bool isUpdatable) {
|
const char *const path, const bool isUpdatable) {
|
||||||
const int fileSize = FileUtils::getFileSize(path);
|
const int fileSize = FileUtils::getFileSize(path);
|
||||||
if (fileSize == -1) {
|
if (fileSize == -1) {
|
||||||
return MmappedBufferPtr(0);
|
return MmappedBufferPtr(nullptr);
|
||||||
} else if (fileSize == 0) {
|
} else if (fileSize == 0) {
|
||||||
return MmappedBufferPtr(new MmappedBuffer(isUpdatable));
|
return MmappedBufferPtr(new MmappedBuffer(isUpdatable));
|
||||||
} else {
|
} else {
|
||||||
|
@ -76,7 +76,7 @@ namespace latinime {
|
||||||
const int filePathLength = snprintf(filePath, filePathBufferSize, "%s%s", dirPath,
|
const int filePathLength = snprintf(filePath, filePathBufferSize, "%s%s", dirPath,
|
||||||
fileName);
|
fileName);
|
||||||
if (filePathLength >= filePathBufferSize) {
|
if (filePathLength >= filePathBufferSize) {
|
||||||
return 0;
|
return MmappedBufferPtr(nullptr);
|
||||||
}
|
}
|
||||||
return openBuffer(filePath, isUpdatable);
|
return openBuffer(filePath, isUpdatable);
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,16 +17,16 @@
|
||||||
#ifndef LATINIME_MMAPPED_BUFFER_H
|
#ifndef LATINIME_MMAPPED_BUFFER_H
|
||||||
#define LATINIME_MMAPPED_BUFFER_H
|
#define LATINIME_MMAPPED_BUFFER_H
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "utils/exclusive_ownership_pointer.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class MmappedBuffer {
|
class MmappedBuffer {
|
||||||
public:
|
public:
|
||||||
typedef ExclusiveOwnershipPointer<MmappedBuffer> MmappedBufferPtr;
|
typedef std::unique_ptr<const MmappedBuffer> MmappedBufferPtr;
|
||||||
|
|
||||||
static MmappedBufferPtr openBuffer(const char *const path,
|
static MmappedBufferPtr openBuffer(const char *const path,
|
||||||
const int bufferOffset, const int bufferSize, const bool isUpdatable);
|
const int bufferOffset, const int bufferSize, const bool isUpdatable);
|
||||||
|
|
|
@ -1,81 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (C) 2013, The Android Open Source Project
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef LATINIME_EXCLUSIVE_OWNERSHIP_POINTER_H
|
|
||||||
#define LATINIME_EXCLUSIVE_OWNERSHIP_POINTER_H
|
|
||||||
|
|
||||||
#include "defines.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
|
||||||
|
|
||||||
template<class T>
|
|
||||||
class ExclusiveOwnershipPointer {
|
|
||||||
public:
|
|
||||||
// This instance become an owner of the raw pointer.
|
|
||||||
AK_FORCE_INLINE ExclusiveOwnershipPointer(T *const rawPointer)
|
|
||||||
: mPointer(rawPointer),
|
|
||||||
mSharedOwnerPtr(new (ExclusiveOwnershipPointer<T> *)(this)) {}
|
|
||||||
|
|
||||||
// Move the ownership.
|
|
||||||
AK_FORCE_INLINE ExclusiveOwnershipPointer(const ExclusiveOwnershipPointer<T> &pointer)
|
|
||||||
: mPointer(pointer.mPointer), mSharedOwnerPtr(pointer.mSharedOwnerPtr) {
|
|
||||||
transferOwnership(&pointer);
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE ~ExclusiveOwnershipPointer() {
|
|
||||||
deletePointersIfHavingOwnership();
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE T *get() const {
|
|
||||||
return mPointer;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
// This class allows to copy and ensures only one instance has the ownership of the
|
|
||||||
// managed pointer.
|
|
||||||
DISALLOW_DEFAULT_CONSTRUCTOR(ExclusiveOwnershipPointer);
|
|
||||||
DISALLOW_ASSIGNMENT_OPERATOR(ExclusiveOwnershipPointer);
|
|
||||||
|
|
||||||
void transferOwnership(const ExclusiveOwnershipPointer<T> *const src) {
|
|
||||||
if (*mSharedOwnerPtr != src) {
|
|
||||||
AKLOGE("Failed to transfer the ownership because src is not the current owner."
|
|
||||||
"src: %p, owner: %p", src, *mSharedOwnerPtr);
|
|
||||||
ASSERT(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Transfer the ownership from src to this instance.
|
|
||||||
*mSharedOwnerPtr = this;
|
|
||||||
}
|
|
||||||
|
|
||||||
void deletePointersIfHavingOwnership() {
|
|
||||||
if (mSharedOwnerPtr && *mSharedOwnerPtr == this) {
|
|
||||||
if (mPointer) {
|
|
||||||
if (DEBUG_DICT) {
|
|
||||||
AKLOGI("Releasing pointer: %p", mPointer);
|
|
||||||
}
|
|
||||||
delete mPointer;
|
|
||||||
}
|
|
||||||
delete mSharedOwnerPtr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
T *mPointer;
|
|
||||||
// mSharedOwnerPtr points a shared memory space where the instance which has the ownership is
|
|
||||||
// stored.
|
|
||||||
ExclusiveOwnershipPointer<T> **mSharedOwnerPtr;
|
|
||||||
};
|
|
||||||
} // namespace latinime
|
|
||||||
#endif /* LATINIME_EXCLUSIVE_OWNERSHIP_POINTER_H */
|
|
Loading…
Reference in New Issue