Merge "Remove BinaryDictionaryInfo and introduce MmapedBuffer."
commit
b0ead57a94
|
@ -18,42 +18,24 @@
|
||||||
|
|
||||||
#include "com_android_inputmethod_latin_BinaryDictionary.h"
|
#include "com_android_inputmethod_latin_BinaryDictionary.h"
|
||||||
|
|
||||||
#include <cerrno>
|
|
||||||
#include <cstring> // for memset()
|
#include <cstring> // for memset()
|
||||||
#include <fcntl.h>
|
|
||||||
#include <sys/mman.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "jni.h"
|
#include "jni.h"
|
||||||
#include "jni_common.h"
|
#include "jni_common.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
#include "suggest/core/dictionary/dictionary.h"
|
#include "suggest/core/dictionary/dictionary.h"
|
||||||
#include "suggest/core/suggest_options.h"
|
#include "suggest/core/suggest_options.h"
|
||||||
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
|
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
|
||||||
#include "utils/autocorrection_threshold_utils.h"
|
#include "utils/autocorrection_threshold_utils.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class ProximityInfo;
|
class ProximityInfo;
|
||||||
|
|
||||||
// Helper method
|
|
||||||
static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) {
|
|
||||||
int ret = munmap(const_cast<void *>(dictBuf), length);
|
|
||||||
if (ret != 0) {
|
|
||||||
AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
|
|
||||||
}
|
|
||||||
ret = close(fd);
|
|
||||||
if (ret != 0) {
|
|
||||||
AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
|
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
|
||||||
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
|
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
|
||||||
PROF_OPEN;
|
PROF_OPEN;
|
||||||
PROF_START(66);
|
PROF_START(66);
|
||||||
// TODO: Move dictionary buffer handling to policyimpl.
|
|
||||||
const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
|
const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
|
||||||
if (sourceDirUtf8Length <= 0) {
|
if (sourceDirUtf8Length <= 0) {
|
||||||
AKLOGE("DICT: Can't get sourceDir string");
|
AKLOGE("DICT: Can't get sourceDir string");
|
||||||
|
@ -62,41 +44,16 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
|
||||||
char sourceDirChars[sourceDirUtf8Length + 1];
|
char sourceDirChars[sourceDirUtf8Length + 1];
|
||||||
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
|
env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
|
||||||
sourceDirChars[sourceDirUtf8Length] = '\0';
|
sourceDirChars[sourceDirUtf8Length] = '\0';
|
||||||
int fd = 0;
|
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
|
||||||
void *dictBuf = 0;
|
DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
|
||||||
int offset = 0;
|
sourceDirChars, static_cast<int>(sourceDirUtf8Length),
|
||||||
const bool updatableMmap = (isUpdatable == JNI_TRUE);
|
static_cast<int>(dictOffset), static_cast<int>(dictSize),
|
||||||
const int openMode = updatableMmap ? O_RDWR : O_RDONLY;
|
isUpdatable == JNI_TRUE);
|
||||||
fd = open(sourceDirChars, openMode);
|
if (!dictionaryStructureWithBufferPolicy) {
|
||||||
if (fd < 0) {
|
|
||||||
AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
int pagesize = getpagesize();
|
|
||||||
offset = static_cast<int>(dictOffset) % pagesize;
|
Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
|
||||||
int adjDictOffset = static_cast<int>(dictOffset) - offset;
|
|
||||||
int adjDictSize = static_cast<int>(dictSize) + offset;
|
|
||||||
const int protMode = updatableMmap ? PROT_READ | PROT_WRITE : PROT_READ;
|
|
||||||
dictBuf = mmap(0, adjDictSize, protMode, MAP_PRIVATE, fd, adjDictOffset);
|
|
||||||
if (dictBuf == MAP_FAILED) {
|
|
||||||
AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
dictBuf = static_cast<char *>(dictBuf) + offset;
|
|
||||||
if (!dictBuf) {
|
|
||||||
AKLOGE("DICT: dictBuf is null");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
Dictionary *dictionary = 0;
|
|
||||||
if (FormatUtils::UNKNOWN_VERSION
|
|
||||||
== FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf),
|
|
||||||
static_cast<int>(dictSize))) {
|
|
||||||
AKLOGE("DICT: dictionary format is unknown, bad magic number");
|
|
||||||
releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd);
|
|
||||||
} else {
|
|
||||||
dictionary = new Dictionary(env, dictBuf, static_cast<int>(dictSize), fd, offset,
|
|
||||||
updatableMmap);
|
|
||||||
}
|
|
||||||
PROF_END(66);
|
PROF_END(66);
|
||||||
PROF_CLOSE;
|
PROF_CLOSE;
|
||||||
return reinterpret_cast<jlong>(dictionary);
|
return reinterpret_cast<jlong>(dictionary);
|
||||||
|
@ -105,13 +62,6 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
|
||||||
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
|
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) return;
|
if (!dictionary) return;
|
||||||
const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo();
|
|
||||||
const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset();
|
|
||||||
const void *dictBuf = binaryDictionaryInfo->getDictBuf();
|
|
||||||
if (!dictBuf) return;
|
|
||||||
releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset,
|
|
||||||
binaryDictionaryInfo->getDictSize() + dictBufOffset,
|
|
||||||
binaryDictionaryInfo->getMmapFd());
|
|
||||||
delete dictionary;
|
delete dictionary;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,65 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (C) 2013, The Android Open Source Project
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef LATINIME_BINARY_DICTIONARY_INFO_H
|
|
||||||
#define LATINIME_BINARY_DICTIONARY_INFO_H
|
|
||||||
|
|
||||||
#include <stdint.h>
|
|
||||||
|
|
||||||
#include "defines.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
|
||||||
|
|
||||||
class BinaryDictionaryInfo {
|
|
||||||
public:
|
|
||||||
AK_FORCE_INLINE BinaryDictionaryInfo(const uint8_t *const dictBuf,
|
|
||||||
const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable)
|
|
||||||
: mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd),
|
|
||||||
mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable) {}
|
|
||||||
|
|
||||||
~BinaryDictionaryInfo() {}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE const uint8_t *getDictBuf() const {
|
|
||||||
return mDictBuf;
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE int getDictSize() const {
|
|
||||||
return mDictSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE int getMmapFd() const {
|
|
||||||
return mMmapFd;
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE int getDictBufOffset() const {
|
|
||||||
return mDictBufOffset;
|
|
||||||
}
|
|
||||||
|
|
||||||
AK_FORCE_INLINE bool isDynamicallyUpdatable() const {
|
|
||||||
return mIsUpdatable;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);
|
|
||||||
|
|
||||||
const uint8_t *const mDictBuf;
|
|
||||||
const int mDictSize;
|
|
||||||
const int mMmapFd;
|
|
||||||
const int mDictBufOffset;
|
|
||||||
const bool mIsUpdatable;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */
|
|
|
@ -23,23 +23,19 @@
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "suggest/core/dictionary/bigram_dictionary.h"
|
#include "suggest/core/dictionary/bigram_dictionary.h"
|
||||||
#include "suggest/core/policy/dictionary_header_structure_policy.h"
|
#include "suggest/core/policy/dictionary_header_structure_policy.h"
|
||||||
|
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
|
||||||
#include "suggest/core/session/dic_traverse_session.h"
|
#include "suggest/core/session/dic_traverse_session.h"
|
||||||
#include "suggest/core/suggest.h"
|
#include "suggest/core/suggest.h"
|
||||||
#include "suggest/core/suggest_options.h"
|
#include "suggest/core/suggest_options.h"
|
||||||
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
|
|
||||||
#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
|
#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
|
||||||
#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
|
#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
|
||||||
#include "utils/log_utils.h"
|
#include "utils/log_utils.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
Dictionary::Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd,
|
Dictionary::Dictionary(JNIEnv *env,
|
||||||
int dictBufOffset, bool isUpdatable)
|
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
|
||||||
: mBinaryDictionaryInfo(static_cast<const uint8_t *>(dict), dictSize, mmapFd,
|
: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
|
||||||
dictBufOffset, isUpdatable),
|
|
||||||
mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory
|
|
||||||
::newDictionaryStructureWithBufferPolicy(
|
|
||||||
static_cast<const uint8_t *>(dict), dictSize)),
|
|
||||||
mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),
|
mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),
|
||||||
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
|
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
|
||||||
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
|
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
|
||||||
|
@ -102,32 +98,17 @@ bool Dictionary::isValidBigram(const int *word0, int length0, const int *word1,
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {
|
void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {
|
||||||
if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
|
|
||||||
// This method should not be called for non-updatable dictionary.
|
|
||||||
AKLOGI("Warning: Dictionary::addUnigramWord() is called for non-updatable dictionary.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability);
|
mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
|
void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,
|
||||||
const int length1, const int probability) {
|
const int length1, const int probability) {
|
||||||
if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
|
|
||||||
// This method should not be called for non-updatable dictionary.
|
|
||||||
AKLOGI("Warning: Dictionary::addBigramWords() is called for non-updatable dictionary.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
|
mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
|
||||||
probability);
|
probability);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::removeBigramWords(const int *const word0, const int length0,
|
void Dictionary::removeBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1) {
|
const int *const word1, const int length1) {
|
||||||
if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
|
|
||||||
// This method should not be called for non-updatable dictionary.
|
|
||||||
AKLOGI("Warning: Dictionary::removeBigramWords() is called for non-updatable dictionary.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
|
mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -155,9 +136,8 @@ void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
|
||||||
dateStringCharBuffer, BUFFER_SIZE);
|
dateStringCharBuffer, BUFFER_SIZE);
|
||||||
|
|
||||||
LogUtils::logToJava(env,
|
LogUtils::logToJava(env,
|
||||||
"Dictionary info: dictionary = %s ; version = %s ; date = %s ; filesize = %i",
|
"Dictionary info: dictionary = %s ; version = %s ; date = %s",
|
||||||
dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer,
|
dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer);
|
||||||
mBinaryDictionaryInfo.getDictSize());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace latinime
|
} // namespace latinime
|
||||||
|
|
|
@ -21,7 +21,6 @@
|
||||||
|
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "jni.h"
|
#include "jni.h"
|
||||||
#include "suggest/core/dictionary/binary_dictionary_info.h"
|
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -54,8 +53,8 @@ class Dictionary {
|
||||||
static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
|
static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
|
||||||
static const int KIND_FLAG_EXACT_MATCH = 0x40000000;
|
static const int KIND_FLAG_EXACT_MATCH = 0x40000000;
|
||||||
|
|
||||||
Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd, int dictBufOffset,
|
Dictionary(JNIEnv *env,
|
||||||
bool isUpdatable);
|
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPoilcy);
|
||||||
|
|
||||||
int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
|
int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
|
||||||
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
|
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
|
||||||
|
@ -78,11 +77,6 @@ class Dictionary {
|
||||||
void removeBigramWords(const int *const word0, const int length0, const int *const word1,
|
void removeBigramWords(const int *const word0, const int length0, const int *const word1,
|
||||||
const int length1);
|
const int length1);
|
||||||
|
|
||||||
// TODO: Remove.
|
|
||||||
const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
|
|
||||||
return &mBinaryDictionaryInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
|
const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
|
||||||
return mDictionaryStructureWithBufferPolicy;
|
return mDictionaryStructureWithBufferPolicy;
|
||||||
}
|
}
|
||||||
|
@ -92,7 +86,6 @@ class Dictionary {
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
|
||||||
|
|
||||||
const BinaryDictionaryInfo mBinaryDictionaryInfo;
|
|
||||||
DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
|
DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
|
||||||
const BigramDictionary *const mBigramDictionary;
|
const BigramDictionary *const mBigramDictionary;
|
||||||
const SuggestInterface *const mGestureSuggest;
|
const SuggestInterface *const mGestureSuggest;
|
||||||
|
|
|
@ -22,18 +22,29 @@
|
||||||
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
|
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/patricia_trie_policy.h"
|
#include "suggest/policyimpl/dictionary/patricia_trie_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
|
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
/* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory
|
/* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory
|
||||||
::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf,
|
::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength,
|
||||||
const int dictSize) {
|
const int bufOffset, const int size, const bool isUpdatable) {
|
||||||
switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) {
|
// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
|
||||||
|
// impl classes of DictionaryStructureWithBufferPolicy.
|
||||||
|
const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset,
|
||||||
|
size, isUpdatable);
|
||||||
|
if (!mmapedBuffer) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
switch (FormatUtils::detectFormatVersion(mmapedBuffer->getBuffer(),
|
||||||
|
mmapedBuffer->getBufferSize())) {
|
||||||
case FormatUtils::VERSION_2:
|
case FormatUtils::VERSION_2:
|
||||||
return new PatriciaTriePolicy(dictBuf);
|
return new PatriciaTriePolicy(mmapedBuffer);
|
||||||
case FormatUtils::VERSION_3:
|
case FormatUtils::VERSION_3:
|
||||||
return new DynamicPatriciaTriePolicy(dictBuf);
|
return new DynamicPatriciaTriePolicy(mmapedBuffer);
|
||||||
default:
|
default:
|
||||||
|
AKLOGE("DICT: dictionary format is unknown, bad magic number");
|
||||||
|
delete mmapedBuffer;
|
||||||
ASSERT(false);
|
ASSERT(false);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,7 +27,8 @@ namespace latinime {
|
||||||
class DictionaryStructureWithBufferPolicyFactory {
|
class DictionaryStructureWithBufferPolicyFactory {
|
||||||
public:
|
public:
|
||||||
static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy(
|
static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy(
|
||||||
const uint8_t *const dictBuf, const int dictSize);
|
const char *const path, const int pathLength, const int bufOffset, const int size,
|
||||||
|
const bool isUpdatable);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);
|
||||||
|
|
|
@ -233,18 +233,30 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const
|
||||||
|
|
||||||
bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length,
|
bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int length,
|
||||||
const int probability) {
|
const int probability) {
|
||||||
|
if (!mBuffer->isUpdatable()) {
|
||||||
|
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
// TODO: Implement.
|
// TODO: Implement.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
|
bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1, const int probability) {
|
const int *const word1, const int length1, const int probability) {
|
||||||
|
if (!mBuffer->isUpdatable()) {
|
||||||
|
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
// TODO: Implement.
|
// TODO: Implement.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
|
bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
|
||||||
const int *const word1, const int length1) {
|
const int *const word1, const int length1) {
|
||||||
|
if (!mBuffer->isUpdatable()) {
|
||||||
|
AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
// TODO: Implement.
|
// TODO: Implement.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,7 @@
|
||||||
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -32,11 +33,14 @@ class DicNodeVector;
|
||||||
|
|
||||||
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
DynamicPatriciaTriePolicy(const uint8_t *const dictBuf)
|
DynamicPatriciaTriePolicy(const MmapedBuffer *const buffer)
|
||||||
: mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()),
|
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
|
||||||
|
mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
|
||||||
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
|
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
|
||||||
|
|
||||||
~DynamicPatriciaTriePolicy() {}
|
~DynamicPatriciaTriePolicy() {
|
||||||
|
delete mBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
AK_FORCE_INLINE int getRootPosition() const {
|
AK_FORCE_INLINE int getRootPosition() const {
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -82,6 +86,7 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
|
||||||
static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
|
static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
|
||||||
|
|
||||||
|
const MmapedBuffer *const mBuffer;
|
||||||
const HeaderPolicy mHeaderPolicy;
|
const HeaderPolicy mHeaderPolicy;
|
||||||
// TODO: Consolidate mDictRoot.
|
// TODO: Consolidate mDictRoot.
|
||||||
const uint8_t *const mDictRoot;
|
const uint8_t *const mDictRoot;
|
||||||
|
|
|
@ -24,6 +24,7 @@
|
||||||
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
#include "suggest/policyimpl/dictionary/header/header_policy.h"
|
||||||
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -32,11 +33,14 @@ class DicNodeVector;
|
||||||
|
|
||||||
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
public:
|
public:
|
||||||
PatriciaTriePolicy(const uint8_t *const dictBuf)
|
PatriciaTriePolicy(const MmapedBuffer *const buffer)
|
||||||
: mHeaderPolicy(dictBuf), mDictRoot(dictBuf + mHeaderPolicy.getSize()),
|
: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
|
||||||
|
mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
|
||||||
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
|
mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
|
||||||
|
|
||||||
~PatriciaTriePolicy() {}
|
~PatriciaTriePolicy() {
|
||||||
|
delete mBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
AK_FORCE_INLINE int getRootPosition() const {
|
AK_FORCE_INLINE int getRootPosition() const {
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -71,25 +75,29 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool addUnigramWord(const int *const word, const int length, const int probability) {
|
bool addUnigramWord(const int *const word, const int length, const int probability) {
|
||||||
// This dictionary format is not updatable.
|
// This method should not be called for non-updatable dictionary.
|
||||||
|
AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool addBigramWords(const int *const word0, const int length0, const int *const word1,
|
bool addBigramWords(const int *const word0, const int length0, const int *const word1,
|
||||||
const int length1, const int probability) {
|
const int length1, const int probability) {
|
||||||
// This dictionary format is not updatable.
|
// This method should not be called for non-updatable dictionary.
|
||||||
|
AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
|
bool removeBigramWords(const int *const word0, const int length0, const int *const word1,
|
||||||
const int length1) {
|
const int length1) {
|
||||||
// This dictionary format is not updatable.
|
// This method should not be called for non-updatable dictionary.
|
||||||
|
AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
|
DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
|
||||||
|
|
||||||
|
const MmapedBuffer *const mBuffer;
|
||||||
const HeaderPolicy mHeaderPolicy;
|
const HeaderPolicy mHeaderPolicy;
|
||||||
const uint8_t *const mDictRoot;
|
const uint8_t *const mDictRoot;
|
||||||
const BigramListPolicy mBigramListPolicy;
|
const BigramListPolicy mBigramListPolicy;
|
||||||
|
|
|
@ -0,0 +1,99 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2013, The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef LATINIME_MMAPED_BUFFER_H
|
||||||
|
#define LATINIME_MMAPED_BUFFER_H
|
||||||
|
|
||||||
|
#include <cerrno>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
#include "defines.h"
|
||||||
|
|
||||||
|
namespace latinime {
|
||||||
|
|
||||||
|
class MmapedBuffer {
|
||||||
|
public:
|
||||||
|
static MmapedBuffer* openBuffer(const char *const path, const int pathLength,
|
||||||
|
const int bufOffset, const int size, const bool isUpdatable) {
|
||||||
|
const int openMode = isUpdatable ? O_RDWR : O_RDONLY;
|
||||||
|
const int fd = open(path, openMode);
|
||||||
|
if (fd < 0) {
|
||||||
|
AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const int pagesize = getpagesize();
|
||||||
|
const int offset = bufOffset % pagesize;
|
||||||
|
int adjOffset = bufOffset - offset;
|
||||||
|
int adjSize = size + offset;
|
||||||
|
const int protMode = isUpdatable ? PROT_READ | PROT_WRITE : PROT_READ;
|
||||||
|
void *const mmapedBuffer = mmap(0, adjSize, protMode, MAP_PRIVATE, fd, adjOffset);
|
||||||
|
if (mmapedBuffer == MAP_FAILED) {
|
||||||
|
AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
|
||||||
|
close(fd);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + offset;
|
||||||
|
if (!buffer) {
|
||||||
|
AKLOGE("DICT: buffer is null");
|
||||||
|
close(fd);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return new MmapedBuffer(buffer, adjSize, fd, offset, isUpdatable);
|
||||||
|
}
|
||||||
|
|
||||||
|
~MmapedBuffer() {
|
||||||
|
int ret = munmap(static_cast<void *>(mBuffer - mBufferOffset),
|
||||||
|
mBufferSize + mBufferOffset);
|
||||||
|
if (ret != 0) {
|
||||||
|
AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
|
||||||
|
}
|
||||||
|
ret = close(mMmapFd);
|
||||||
|
if (ret != 0) {
|
||||||
|
AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AK_FORCE_INLINE uint8_t *getBuffer() const {
|
||||||
|
return mBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
AK_FORCE_INLINE int getBufferSize() const {
|
||||||
|
return mBufferSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
AK_FORCE_INLINE bool isUpdatable() const {
|
||||||
|
return mIsUpdatable;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
AK_FORCE_INLINE MmapedBuffer(uint8_t *const buffer, const int bufferSize, const int mmapFd,
|
||||||
|
const int bufferOffset, const bool isUpdatable)
|
||||||
|
: mBuffer(buffer), mBufferSize(bufferSize), mMmapFd(mmapFd),
|
||||||
|
mBufferOffset(bufferOffset), mIsUpdatable(isUpdatable) {}
|
||||||
|
|
||||||
|
DISALLOW_IMPLICIT_CONSTRUCTORS(MmapedBuffer);
|
||||||
|
|
||||||
|
uint8_t *const mBuffer;
|
||||||
|
const int mBufferSize;
|
||||||
|
const int mMmapFd;
|
||||||
|
const int mBufferOffset;
|
||||||
|
const bool mIsUpdatable;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
#endif /* LATINIME_MMAPED_BUFFER_H */
|
Loading…
Reference in New Issue