nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Use BinaryDictonaryInfo instead of raw pointers."

main
Keisuke Kuroyanagi 2013-06-03 00:51:43 +00:00 committed by Android (Google) Code Review
parent 88ad30f40b 0ecfb94247
commit 712fefd85f
22 changed files with 571 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
native/jni/Android.mk
View File

@ -58,6 +58,8 @@ LATIN_IME_CORE_SRC_FILES := \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
char_utils.cpp \
binary_dictionary_format.cpp \
byte_array_utils.cpp \
dictionary.cpp \
digraph_utils.cpp) \
$(addprefix suggest/core/layout/, \

9
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -35,7 +35,8 @@
#include "jni.h"
#include "jni_common.h"
#include "suggest_options.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/binary_dictionary_format.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/dictionary.h"
namespace latinime {
@ -110,8 +111,8 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
return 0;
}
Dictionary *dictionary = 0;
if (BinaryFormat::UNKNOWN_FORMAT
== BinaryFormat::detectFormat(static_cast<uint8_t *>(dictBuf),
if (BinaryDictionaryFormat::UNKNOWN_VERSION
== BinaryDictionaryFormat::detectFormatVersion(static_cast<uint8_t *>(dictBuf),
static_cast<int>(dictSize))) {
AKLOGE("DICT: dictionary format is unknown, bad magic number");
#ifdef USE_MMAP_FOR_DICTIONARY
@ -260,7 +261,7 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
const void *dictBuf = dictionary->getDict();
const void *dictBuf = dictionary->getBinaryDictionaryInfo()->getDictBuf();
if (!dictBuf) return;
#ifdef USE_MMAP_FOR_DICTIONARY
releaseDictBuf(static_cast<const char *>(dictBuf) - dictionary->getDictBufAdjust(),

12
native/jni/src/bigram_dictionary.cpp
View File

@ -21,6 +21,7 @@
#include "bigram_dictionary.h"
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/bloom_filter.h"
#include "suggest/core/dictionary/char_utils.h"
@ -29,7 +30,8 @@
namespace latinime {
BigramDictionary::BigramDictionary(const uint8_t *const streamStart) : DICT_ROOT(streamStart) {
BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo)
: mBinaryDictionaryInfo(binaryDictionaryInfo) {
if (DEBUG_DICT) {
AKLOGI("BigramDictionary - constructor");
}
@ -104,7 +106,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i
// TODO: remove unused arguments, and refrain from storing stuff in members of this class
// TODO: have "in" arguments before "out" ones, and make out args explicit in the name
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
@ -150,7 +152,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i
int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const {
if (0 >= prevWordLength) return 0;
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength,
forceLowerCaseSearch);
@ -171,7 +173,7 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in
void BigramDictionary::fillBigramAddressToProbabilityMapAndFilter(const int *prevWord,
const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const {
memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE);
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
if (0 == pos) {
@ -210,7 +212,7 @@ bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) cons
bool BigramDictionary::isValidBigram(const int *word1, int length1, const int *word2,
int length2) const {
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(word1, length1, false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
if (0 == pos) return false;

8
native/jni/src/bigram_dictionary.h
View File

@ -24,9 +24,12 @@
namespace latinime {
class BinaryDictionaryInfo;
class BigramDictionary {
public:
BigramDictionary(const uint8_t *const streamStart);
BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo);
int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords,
int *frequencies, int *outputTypes) const;
void fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength,
@ -35,13 +38,14 @@ class BigramDictionary {
~BigramDictionary();
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary);
void addWordBigram(int *word, int length, int probability, int *bigramProbability,
int *bigramCodePoints, int *outputTypes) const;
bool checkFirstCharacter(int *word, int *inputCodePoints) const;
int getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const;
const uint8_t *const DICT_ROOT;
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
// TODO: Re-implement proximity correction for bigram correction
static const int MAX_ALTERNATIVES = 1;
};

92
native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
View File

@ -20,6 +20,7 @@
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_utils.h"
#include "suggest/core/dicnode/dic_node_vector.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/char_utils.h"
#include "suggest/core/dictionary/multi_bigram_map.h"
@ -33,20 +34,23 @@ namespace latinime {
// Node initialization utils //
///////////////////////////////
/* static */ void DicNodeUtils::initAsRoot(const int rootPos, const uint8_t *const dicRoot,
const int prevWordNodePos, DicNode *newRootNode) {
int curPos = rootPos;
/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int prevWordNodePos, DicNode *const newRootNode) {
int curPos = binaryDictionaryInfo->getRootPosition();
const int pos = curPos;
const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos);
const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &curPos);
const int childrenPos = curPos;
newRootNode->initAsRoot(pos, childrenPos, childrenCount, prevWordNodePos);
}
/*static */ void DicNodeUtils::initAsRootWithPreviousWord(const int rootPos,
const uint8_t *const dicRoot, DicNode *prevWordLastNode, DicNode *newRootNode) {
int curPos = rootPos;
/*static */ void DicNodeUtils::initAsRootWithPreviousWord(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
DicNode *const prevWordLastNode, DicNode *const newRootNode) {
int curPos = binaryDictionaryInfo->getRootPosition();
const int pos = curPos;
const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos);
const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &curPos);
const int childrenPos = curPos;
newRootNode->initAsRootWithPreviousWord(prevWordLastNode, pos, childrenPos, childrenCount);
}
@ -72,16 +76,19 @@ namespace latinime {
}
/* static */ int DicNodeUtils::createAndGetLeavingChildNode(DicNode *dicNode, int pos,
const uint8_t *const dicRoot, const int terminalDepth, const ProximityInfoState *pInfoState,
const int pointIndex, const bool exactOnly, const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) {
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth,
const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo,
DicNodeVector *childDicNodes) {
int nextPos = pos;
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dicRoot, &pos);
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &pos);
const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
const bool isTerminal = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags));
const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags);
int codePoint = BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos);
int codePoint = BinaryFormat::getCodePointAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &pos);
ASSERT(NOT_A_CODE_POINT != codePoint);
const int nodeCodePoint = codePoint;
// TODO: optimize this
@ -91,7 +98,8 @@ namespace latinime {
do {
const int nextCodePoint = hasMultipleChars
? BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos) : NOT_A_CODE_POINT;
? BinaryFormat::getCodePointAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT;
const bool isLastChar = (NOT_A_CODE_POINT == nextCodePoint);
if (!isLastChar) {
additionalWordBuf[additionalSubwordLength++] = nextCodePoint;
@ -99,12 +107,14 @@ namespace latinime {
codePoint = nextCodePoint;
} while (NOT_A_CODE_POINT != codePoint);
const int probability =
isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(dicRoot, pos) : -1;
const int probability = isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(
binaryDictionaryInfo->getDictRoot(), pos) : -1;
pos = BinaryFormat::skipProbability(flags, pos);
int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(dicRoot, flags, pos) : 0;
int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(
binaryDictionaryInfo->getDictRoot(), flags, pos) : 0;
const int attributesPos = BinaryFormat::skipChildrenPosition(flags, pos);
const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(dicRoot, flags, pos);
const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(
binaryDictionaryInfo->getDictRoot(), flags, pos);
if (isDicNodeFilteredOut(nodeCodePoint, pInfo, codePointsFilter)) {
return siblingPos;
@ -112,8 +122,8 @@ namespace latinime {
if (!isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, nodeCodePoint)) {
return siblingPos;
}
const int childrenCount = hasChildren
? BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &childrenPos) : 0;
const int childrenCount = hasChildren ? BinaryFormat::getGroupCountAndForwardPointer(
binaryDictionaryInfo->getDictRoot(), &childrenPos) : 0;
childDicNodes->pushLeavingChild(dicNode, nextPos, flags, childrenPos, attributesPos, siblingPos,
nodeCodePoint, childrenCount, probability, -1 /* bigramProbability */, isTerminal,
hasMultipleChars, hasChildren, additionalSubwordLength, additionalWordBuf);
@ -149,16 +159,18 @@ namespace latinime {
}
/* static */ void DicNodeUtils::createAndGetAllLeavingChildNodes(DicNode *dicNode,
const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex,
const bool exactOnly, const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) {
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo,
DicNodeVector *childDicNodes) {
const int terminalDepth = dicNode->getLeavingDepth();
const int childCount = dicNode->getChildrenCount();
int nextPos = dicNode->getChildrenPos();
for (int i = 0; i < childCount; i++) {
const int filterSize = codePointsFilter ? codePointsFilter->size() : 0;
nextPos = createAndGetLeavingChildNode(dicNode, nextPos, dicRoot, terminalDepth, pInfoState,
pointIndex, exactOnly, codePointsFilter, pInfo, childDicNodes);
nextPos = createAndGetLeavingChildNode(dicNode, nextPos, binaryDictionaryInfo,
terminalDepth, pInfoState, pointIndex, exactOnly, codePointsFilter, pInfo,
childDicNodes);
if (!pInfo && filterSize > 0 && childDicNodes->exceeds(filterSize)) {
// All code points have been found.
break;
@ -166,14 +178,15 @@ namespace latinime {
}
}
/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
DicNodeVector *childDicNodes) {
getProximityChildDicNodes(dicNode, dicRoot, 0, 0, false, childDicNodes);
/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode,
const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) {
getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes);
}
/* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode,
const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex,
bool exactOnly, DicNodeVector *childDicNodes) {
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
DicNodeVector *childDicNodes) {
if (dicNode->isTotalInputSizeExceedingLimit()) {
return;
}
@ -181,9 +194,9 @@ namespace latinime {
DicNodeUtils::createAndGetPassingChildNode(dicNode, pInfoState, pointIndex, exactOnly,
childDicNodes);
} else {
DicNodeUtils::createAndGetAllLeavingChildNodes(dicNode, dicRoot, pInfoState, pointIndex,
exactOnly, 0 /* codePointsFilter */, 0 /* pInfo */,
childDicNodes);
DicNodeUtils::createAndGetAllLeavingChildNodes(
dicNode, binaryDictionaryInfo, pInfoState, pointIndex, exactOnly,
0 /* codePointsFilter */, 0 /* pInfo */, childDicNodes);
}
}
@ -193,19 +206,21 @@ namespace latinime {
/**
* Computes the combined bigram / unigram cost for the given dicNode.
*/
/* static */ float DicNodeUtils::getBigramNodeImprobability(const uint8_t *const dicRoot,
/* static */ float DicNodeUtils::getBigramNodeImprobability(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *multiBigramMap) {
if (node->isImpossibleBigramWord()) {
return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
const int probability = getBigramNodeProbability(dicRoot, node, multiBigramMap);
const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap);
// TODO: This equation to calculate the improbability looks unreasonable. Investigate this.
const float cost = static_cast<float>(MAX_PROBABILITY - probability)
/ static_cast<float>(MAX_PROBABILITY);
return cost;
}
/* static */ int DicNodeUtils::getBigramNodeProbability(const uint8_t *const dicRoot,
/* static */ int DicNodeUtils::getBigramNodeProbability(
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *multiBigramMap) {
const int unigramProbability = node->getProbability();
const int wordPos = node->getPos();
@ -216,9 +231,10 @@ namespace latinime {
}
if (multiBigramMap) {
return multiBigramMap->getBigramProbability(
dicRoot, prevWordPos, wordPos, unigramProbability);
binaryDictionaryInfo, prevWordPos, wordPos, unigramProbability);
}
return BinaryFormat::getBigramProbability(dicRoot, prevWordPos, wordPos, unigramProbability);
return BinaryFormat::getBigramProbability(
binaryDictionaryInfo->getDictRoot(), prevWordPos, wordPos, unigramProbability);
}
///////////////////////////////////////

26
native/jni/src/suggest/core/dicnode/dic_node_utils.h
View File

@ -24,6 +24,7 @@
namespace latinime {
class BinaryDictionaryInfo;
class DicNode;
class DicNodeVector;
class ProximityInfo;
@ -34,19 +35,20 @@ class DicNodeUtils {
public:
static int appendTwoWords(const int *src0, const int16_t length0, const int *src1,
const int16_t length1, int *dest);
static void initAsRoot(const int rootPos, const uint8_t *const dicRoot,
static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const int prevWordNodePos, DicNode *newRootNode);
static void initAsRootWithPreviousWord(const int rootPos, const uint8_t *const dicRoot,
static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo,
DicNode *prevWordLastNode, DicNode *newRootNode);
static void initByCopy(DicNode *srcNode, DicNode *destNode);
static void getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
DicNodeVector *childDicNodes);
static float getBigramNodeImprobability(const uint8_t *const dicRoot,
static void getAllChildDicNodes(DicNode *dicNode,
const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes);
static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *const multiBigramMap);
static bool isDicNodeFilteredOut(const int nodeCodePoint, const ProximityInfo *const pInfo,
const std::vector<int> *const codePointsFilter);
// TODO: Move to private
static void getProximityChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot,
static void getProximityChildDicNodes(DicNode *dicNode,
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
DicNodeVector *childDicNodes);
@ -60,16 +62,18 @@ class DicNodeUtils {
// Max number of bigrams to look up
static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500;
static int getBigramNodeProbability(const uint8_t *const dicRoot, const DicNode *const node,
MultiBigramMap *multiBigramMap);
static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
const DicNode *const node, MultiBigramMap *multiBigramMap);
static void createAndGetPassingChildNode(DicNode *dicNode, const ProximityInfoState *pInfoState,
const int pointIndex, const bool exactOnly, DicNodeVector *childDicNodes);
static void createAndGetAllLeavingChildNodes(DicNode *dicNode, const uint8_t *const dicRoot,
static void createAndGetAllLeavingChildNodes(DicNode *dicNode,
const BinaryDictionaryInfo *const binaryDictionaryInfo,
const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly,
const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes);
static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, const uint8_t *const dicRoot,
const int terminalDepth, const ProximityInfoState *pInfoState, const int pointIndex,
static int createAndGetLeavingChildNode(DicNode *dicNode, int pos,
const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth,
const ProximityInfoState *pInfoState, const int pointIndex,
const bool exactOnly, const std::vector<int> *const codePointsFilter,
const ProximityInfo *const pInfo, DicNodeVector *childDicNodes);

84
native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/core/dictionary/binary_dictionary_format.h"
namespace latinime {
/**
* Dictionary size
*/
// Any file smaller than this is not a dictionary.
const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4;
/**
* Format versions
*/
// Originally, format version 1 had a 16-bit magic number, then the version number `01'
// then options that must be 0. Hence the first 32-bits of the format are always as follow
// and it's okay to consider them a magic number as a whole.
const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100;
const int BinaryDictionaryFormat::FORMAT_VERSION_1_HEADER_SIZE = 5;
// The versions of Latin IME that only handle format version 1 only test for the magic
// number, so we had to change it so that version 2 files would be rejected by older
// implementations. On this occasion, we made the magic number 32 bits long.
const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE;
// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12
const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12;
const int BinaryDictionaryFormat::VERSION_2_MAGIC_NUMBER_SIZE = 4;
const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_VERSION_SIZE = 2;
const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_FLAG_SIZE = 2;
/* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion(
const uint8_t *const dict, const int dictSize) {
// The magic number is stored big-endian.
// If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't
// understand this format.
if (dictSize < DICTIONARY_MINIMUM_SIZE) {
return UNKNOWN_VERSION;
}
const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
switch (magicNumber) {
case FORMAT_VERSION_1_MAGIC_NUMBER:
// Format 1 header is exactly 5 bytes long and looks like:
// Magic number (2 bytes) 0x78 0xB1
// Version number (1 byte) 0x01
// Options (2 bytes) must be 0x00 0x00
return VERSION_1;
case FORMAT_VERSION_2_MAGIC_NUMBER:
// Version 2 dictionaries are at least 12 bytes long.
// If this dictionary has the version 2 magic number but is less than 12 bytes long,
// then it's an unknown format and we need to avoid confidently reading the next bytes.
if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) {
return UNKNOWN_VERSION;
}
// Format 2 header is as follows:
// Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE
// Version number (2 bytes) 0x00 0x02
// Options (2 bytes)
// Header size (4 bytes) : integer, big endian
if (ByteArrayUtils::readUint16(dict, 4) == 2) {
return VERSION_2;
} else {
return UNKNOWN_VERSION;
}
default:
return UNKNOWN_VERSION;
}
}
} // namespace latinime

71
native/jni/src/suggest/core/dictionary/binary_dictionary_format.h Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_BINARY_DICTIONARY_FORMAT_H
#define LATINIME_BINARY_DICTIONARY_FORMAT_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
/**
* Methods to handle binary dictionary format version.
*
* Currently, we have a file with a similar name, binary_format.h. binary_format.h contains binary
* reading methods and utility methods for various purposes.
* On the other hand, this file deals with only about dictionary format version.
*/
class BinaryDictionaryFormat {
public:
// TODO: Remove obsolete version logic
enum FORMAT_VERSION {
VERSION_1,
VERSION_2,
UNKNOWN_VERSION
};
static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize);
static AK_FORCE_INLINE int getHeaderSize(
const uint8_t *const dict, const FORMAT_VERSION format) {
switch (format) {
case VERSION_1:
return FORMAT_VERSION_1_HEADER_SIZE;
case VERSION_2:
// See the format of the header in the comment in detectFormat() above
return ByteArrayUtils::readUint32(dict, 8);
default:
return S_INT_MAX;
}
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat);
static const int DICTIONARY_MINIMUM_SIZE;
static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER;
static const int FORMAT_VERSION_1_HEADER_SIZE;
static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER;
static const int FORMAT_VERSION_2_MINIMUM_SIZE;
static const int VERSION_2_MAGIC_NUMBER_SIZE;
static const int VERSION_2_DICTIONARY_VERSION_SIZE ;
static const int VERSION_2_DICTIONARY_FLAG_SIZE;
};
} // namespace latinime
#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_H */

58
native/jni/src/suggest/core/dictionary/binary_dictionary_info.h Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_BINARY_DICTIONARY_INFO_H
#define LATINIME_BINARY_DICTIONARY_INFO_H
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_format.h"
namespace latinime {
class BinaryDictionaryInfo {
public:
BinaryDictionaryInfo(const uint8_t *const dictBuf, const int dictSize)
: mDictBuf(dictBuf),
mFormat(BinaryDictionaryFormat::detectFormatVersion(mDictBuf, dictSize)),
mDictRoot(mDictBuf + BinaryDictionaryFormat::getHeaderSize(mDictBuf, mFormat)) {}
AK_FORCE_INLINE const uint8_t *getDictBuf() const {
return mDictBuf;
}
AK_FORCE_INLINE const uint8_t *getDictRoot() const {
return mDictRoot;
}
AK_FORCE_INLINE BinaryDictionaryFormat::FORMAT_VERSION getFormat() const {
return mFormat;
}
AK_FORCE_INLINE int getRootPosition() const {
return 0;
}
private:
DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);
const uint8_t *const mDictBuf;
const BinaryDictionaryFormat::FORMAT_VERSION mFormat;
const uint8_t *const mDictRoot;
};
}
#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */

24
native/jni/src/suggest/core/dictionary/byte_array_utils.cpp Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
const uint8_t ByteArrayUtils::MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
const uint8_t ByteArrayUtils::CHARACTER_ARRAY_TERMINATOR = 0x1F;
} // namespace latinime

148
native/jni/src/suggest/core/dictionary/byte_array_utils.h Normal file
View File

@ -0,0 +1,148 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_BYTE_ARRAY_UTILS_H
#define LATINIME_BYTE_ARRAY_UTILS_H
#include <stdint.h>
#include "defines.h"
namespace latinime {
/**
* Utility methods for reading byte arrays.
*/
class ByteArrayUtils {
public:
/**
* Integer
*
* Each method read a corresponding size integer in a big endian manner.
*/
static AK_FORCE_INLINE uint32_t readUint32(const uint8_t *const buffer, const int pos) {
return (buffer[pos] << 24) ^ (buffer[pos + 1] << 16)
^ (buffer[pos + 2] << 8) ^ buffer[pos + 3];
}
static AK_FORCE_INLINE uint32_t readUint24(const uint8_t *const buffer, const int pos) {
return (buffer[pos] << 16) ^ (buffer[pos + 1] << 8) ^ buffer[pos + 2];
}
static AK_FORCE_INLINE uint16_t readUint16(const uint8_t *const buffer, const int pos) {
return (buffer[pos] << 8) ^ buffer[pos + 1];
}
static AK_FORCE_INLINE uint8_t readUint8(const uint8_t *const buffer, const int pos) {
return buffer[pos];
}
static AK_FORCE_INLINE uint32_t readUint32andAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const uint32_t value = readUint32(buffer, *pos);
*pos += 4;
return value;
}
static AK_FORCE_INLINE uint32_t readUint24andAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const uint32_t value = readUint24(buffer, *pos);
*pos += 3;
return value;
}
static AK_FORCE_INLINE uint16_t readUint16andAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const uint16_t value = readUint16(buffer, *pos);
*pos += 2;
return value;
}
static AK_FORCE_INLINE uint8_t readUint8andAdvancePosition(
const uint8_t *const buffer, int *const pos) {
return buffer[(*pos)++];
}
/**
* Code Point
*
* 1 byte = bbbbbbbb match
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
* else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
* unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
* 00011111 would be outside unicode.
* else: iso-latin-1 code
* This allows for the whole unicode range to be encoded, including chars outside of
* the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
* characters which should never happen anyway (and still work, but take 3 bytes).
*/
static AK_FORCE_INLINE int readCodePoint(const uint8_t *const buffer, const int pos) {
int p = pos;
return readCodePointAndAdvancePosition(buffer, &p);
}
static AK_FORCE_INLINE int readCodePointAndAdvancePosition(
const uint8_t *const buffer, int *const pos) {
const uint8_t firstByte = readUint8(buffer, *pos);
if (firstByte < MINIMAL_ONE_BYTE_CHARACTER_VALUE) {
if (firstByte == CHARACTER_ARRAY_TERMINATOR) {
*pos += 1;
return NOT_A_CODE_POINT;
} else {
return readUint24andAdvancePosition(buffer, pos);
}
} else {
*pos += 1;
return firstByte;
}
}
/**
* String (array of code points)
*
* Reads code points until the terminator is found.
*/
// Returns the length of the string.
static int readStringAndAdvancePosition(const uint8_t *const buffer, int *const pos,
int *const outBuffer, const int maxLength) {
int length = 0;
int codePoint = readCodePointAndAdvancePosition(buffer, pos);
while (NOT_A_CODE_POINT != codePoint && length < maxLength) {
outBuffer[length++] = codePoint;
codePoint = readCodePointAndAdvancePosition(buffer, pos);
}
return length;
}
// Advances the position and returns the length of the string.
static int advancePositionToBehindString(
const uint8_t *const buffer, int *const pos, const int maxLength) {
int length = 0;
int codePoint = readCodePointAndAdvancePosition(buffer, pos);
while (NOT_A_CODE_POINT != codePoint && length < maxLength) {
codePoint = readCodePointAndAdvancePosition(buffer, pos);
}
return length;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils);
static const uint8_t MINIMAL_ONE_BYTE_CHARACTER_VALUE;
static const uint8_t CHARACTER_ARRAY_TERMINATOR;
};
} // namespace latinime
#endif /* LATINIME_BYTE_ARRAY_UTILS_H */

10
native/jni/src/suggest/core/dictionary/dictionary.cpp
View File

@ -34,13 +34,11 @@
namespace latinime {
Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust)
: mDict(static_cast<unsigned char *>(dict)),
mOffsetDict((static_cast<unsigned char *>(dict))
+ BinaryFormat::getHeaderSize(mDict, dictSize)),
: mBinaryDicitonaryInfo(static_cast<const uint8_t *>(dict), dictSize),
mDictSize(dictSize), mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust),
mUnigramDictionary(new UnigramDictionary(mOffsetDict,
BinaryFormat::getFlags(mDict, dictSize))),
mBigramDictionary(new BigramDictionary(mOffsetDict)),
mUnigramDictionary(new UnigramDictionary(&mBinaryDicitonaryInfo,
BinaryFormat::getFlags(mBinaryDicitonaryInfo.getDictBuf(), dictSize))),
mBigramDictionary(new BigramDictionary(&mBinaryDicitonaryInfo)),
mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
}

11
native/jni/src/suggest/core/dictionary/dictionary.h
View File

@ -20,6 +20,7 @@
#include <stdint.h>
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
namespace latinime {
@ -64,11 +65,8 @@ class Dictionary {
int getProbability(const int *word, int length) const;
bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const;
const uint8_t *getDict() const { // required to release dictionary buffer
return mDict;
}
const uint8_t *getOffsetDict() const {
return mOffsetDict;
const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
return &mBinaryDicitonaryInfo;
}
int getDictSize() const { return mDictSize; }
int getMmapFd() const { return mMmapFd; }
@ -78,9 +76,8 @@ class Dictionary {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
const uint8_t *mDict;
const uint8_t *mOffsetDict;
const BinaryDictionaryInfo mBinaryDicitonaryInfo;
// Used only for the mmap version of dictionary loading, but we use these as dummy variables
// also for the malloc version.
const int mDictSize;

23
native/jni/src/suggest/core/dictionary/multi_bigram_map.h
View File

@ -17,10 +17,9 @@
#ifndef LATINIME_MULTI_BIGRAM_MAP_H
#define LATINIME_MULTI_BIGRAM_MAP_H
#include <stdint.h>
#include "defines.h"
#include "hash_map_compat.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
namespace latinime {
@ -35,20 +34,20 @@ class MultiBigramMap {
// Look up the bigram probability for the given word pair from the cached bigram maps.
// Also caches the bigrams if there is space remaining and they have not been cached already.
int getBigramProbability(const uint8_t *const dicRoot, const int wordPosition,
const int nextWordPosition, const int unigramProbability) {
int getBigramProbability(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
const int wordPosition, const int nextWordPosition, const int unigramProbability) {
hash_map_compat<int, BigramMap>::const_iterator mapPosition =
mBigramMaps.find(wordPosition);
if (mapPosition != mBigramMaps.end()) {
return mapPosition->second.getBigramProbability(nextWordPosition, unigramProbability);
}
if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
addBigramsForWordPosition(dicRoot, wordPosition);
addBigramsForWordPosition(binaryDicitonaryInfo, wordPosition);
return mBigramMaps[wordPosition].getBigramProbability(
nextWordPosition, unigramProbability);
}
return BinaryFormat::getBigramProbability(
dicRoot, wordPosition, nextWordPosition, unigramProbability);
return BinaryFormat::getBigramProbability(binaryDicitonaryInfo->getDictRoot(),
wordPosition, nextWordPosition, unigramProbability);
}
void clear() {
@ -63,8 +62,9 @@ class MultiBigramMap {
BigramMap() : mBigramMap(DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP) {}
~BigramMap() {}
void init(const uint8_t *const dicRoot, int position) {
BinaryFormat::fillBigramProbabilityToHashMap(dicRoot, position, &mBigramMap);
void init(const BinaryDictionaryInfo *const binaryDicitonaryInfo, const int position) {
BinaryFormat::fillBigramProbabilityToHashMap(
binaryDicitonaryInfo->getDictRoot(), position, &mBigramMap);
}
inline int getBigramProbability(const int nextWordPosition, const int unigramProbability)
@ -78,8 +78,9 @@ class MultiBigramMap {
hash_map_compat<int, int> mBigramMap;
};
void addBigramsForWordPosition(const uint8_t *const dicRoot, const int position) {
mBigramMaps[position].init(dicRoot, position);
void addBigramsForWordPosition(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
const int position) {
mBigramMaps[position].init(binaryDicitonaryInfo, position);
}
hash_map_compat<int, BigramMap> mBigramMaps;

24
native/jni/src/suggest/core/dictionary/terminal_attributes.h
View File

@ -19,6 +19,7 @@
#include <stdint.h>
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
namespace latinime {
@ -32,8 +33,9 @@ class TerminalAttributes {
public:
class ShortcutIterator {
public:
ShortcutIterator(const uint8_t *dict, const int pos, const uint8_t flags)
: mDict(dict), mPos(pos),
ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int pos,
const uint8_t flags)
: mBinaryDicitionaryInfo(binaryDictionaryInfo), mPos(pos),
mHasNextShortcutTarget(0 != (flags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS)) {
}
@ -44,11 +46,13 @@ class TerminalAttributes {
// Gets the shortcut target itself as an int string. For parameters and return value
// see BinaryFormat::getWordAtAddress.
inline int getNextShortcutTarget(const int maxDepth, int *outWord, int *outFreq) {
const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos);
const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(
mBinaryDicitionaryInfo->getDictRoot(), &mPos);
mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT);
unsigned int i;
for (i = 0; i < MAX_WORD_LENGTH; ++i) {
const int codePoint = BinaryFormat::getCodePointAndForwardPointer(mDict, &mPos);
const int codePoint = BinaryFormat::getCodePointAndForwardPointer(
mBinaryDicitionaryInfo->getDictRoot(), &mPos);
if (NOT_A_CODE_POINT == codePoint) break;
outWord[i] = codePoint;
}
@ -57,19 +61,21 @@ class TerminalAttributes {
}
private:
const uint8_t *const mDict;
const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
int mPos;
bool mHasNextShortcutTarget;
};
TerminalAttributes(const uint8_t *const dict, const uint8_t flags, const int pos)
: mDict(dict), mFlags(flags), mStartPos(pos) {
TerminalAttributes(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
const uint8_t flags, const int pos)
: mBinaryDicitionaryInfo(binaryDicitonaryInfo), mFlags(flags), mStartPos(pos) {
}
inline ShortcutIterator getShortcutIterator() const {
// The size of the shortcuts is stored here so that the whole shortcut chunk can be
// skipped quickly, so we ignore it.
return ShortcutIterator(mDict, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags);
return ShortcutIterator(
mBinaryDicitionaryInfo, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags);
}
bool isBlacklistedOrNotAWord() const {
@ -78,7 +84,7 @@ class TerminalAttributes {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes);
const uint8_t *const mDict;
const BinaryDictionaryInfo *const mBinaryDicitionaryInfo;
const uint8_t mFlags;
const int mStartPos;
};

2
native/jni/src/suggest/core/policy/weighting.cpp
View File

@ -142,7 +142,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n
case CT_TERMINAL: {
const float languageImprobability =
DicNodeUtils::getBigramNodeImprobability(
traverseSession->getOffsetDict(), dicNode, multiBigramMap);
traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap);
return weighting->getTerminalLanguageCost(traverseSession, dicNode, languageImprobability);
}
case CT_NEW_WORD_SPACE_SUBSTITUTION:

14
native/jni/src/suggest/core/session/dic_traverse_session.cpp
View File

@ -20,6 +20,7 @@
#include "dic_traverse_wrapper.h"
#include "jni.h"
#include "suggest/core/dicnode/dic_node_utils.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/dictionary.h"
@ -65,7 +66,8 @@ static TraverseSessionFactoryRegisterer traverseSessionFactoryRegisterer;
void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord,
int prevWordLength, const SuggestOptions *const suggestOptions) {
mDictionary = dictionary;
mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(mDictionary->getDict(),
mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(
mDictionary->getBinaryDictionaryInfo()->getDictBuf(),
mDictionary->getDictSize());
mSuggestOptions = suggestOptions;
if (!prevWord) {
@ -73,12 +75,14 @@ void DicTraverseSession::init(const Dictionary *const dictionary, const int *pre
return;
}
// TODO: merge following similar calls to getTerminalPosition into one case-insensitive call.
mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord,
mPrevWordPos = BinaryFormat::getTerminalPosition(
dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord,
prevWordLength, false /* forceLowerCaseSearch */);
if (mPrevWordPos == NOT_VALID_WORD) {
// Check bigrams for lower-cased previous word if original was not found. Useful for
// auto-capitalized words like "The [current_word]".
mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord,
mPrevWordPos = BinaryFormat::getTerminalPosition(
dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord,
prevWordLength, true /* forceLowerCaseSearch */);
}
}
@ -93,8 +97,8 @@ void DicTraverseSession::setupForGetSuggestions(const ProximityInfo *pInfo,
maxSpatialDistance, maxPointerCount);
}
const uint8_t *DicTraverseSession::getOffsetDict() const {
return mDictionary->getOffsetDict();
const BinaryDictionaryInfo *DicTraverseSession::getBinaryDictionaryInfo() const {
return mDictionary->getBinaryDictionaryInfo();
}
int DicTraverseSession::getDictFlags() const {

3
native/jni/src/suggest/core/session/dic_traverse_session.h
View File

@ -28,6 +28,7 @@
namespace latinime {
class BinaryDictionaryInfo;
class Dictionary;
class ProximityInfo;
class SuggestOptions;
@ -56,7 +57,7 @@ class DicTraverseSession {
void resetCache(const int nextActiveCacheSize, const int maxWords);
// TODO: Remove
const uint8_t *getOffsetDict() const;
const BinaryDictionaryInfo *getBinaryDictionaryInfo() const;
int getDictFlags() const;
//--------------------

21
native/jni/src/suggest/core/suggest.cpp
View File

@ -105,8 +105,8 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession, int commitPo
traverseSession->resetCache(TRAVERSAL->getMaxCacheSize(), MAX_RESULTS);
// Create a new dic node here
DicNode rootNode;
DicNodeUtils::initAsRoot(traverseSession->getDicRootPos(),
traverseSession->getOffsetDict(), traverseSession->getPrevWordPos(), &rootNode);
DicNodeUtils::initAsRoot(traverseSession->getBinaryDictionaryInfo(),
traverseSession->getPrevWordPos(), &rootNode);
traverseSession->getDicTraverseCache()->copyPushActive(&rootNode);
}
}
@ -158,7 +158,7 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen
terminalIndex, doubleLetterTerminalIndex, doubleLetterLevel);
const float compoundDistance = terminalDicNode->getCompoundDistance(languageWeight)
+ doubleLetterCost;
const TerminalAttributes terminalAttributes(traverseSession->getOffsetDict(),
const TerminalAttributes terminalAttributes(traverseSession->getBinaryDictionaryInfo(),
terminalDicNode->getFlags(), terminalDicNode->getAttributesPos());
const bool isPossiblyOffensiveWord = terminalDicNode->getProbability() <= 0;
const bool isExactMatch = terminalDicNode->isExactMatch();
@ -284,7 +284,7 @@ void Suggest::expandCurrentDicNodes(DicTraverseSession *traverseSession) const {
}
DicNodeUtils::getAllChildDicNodes(
&dicNode, traverseSession->getOffsetDict(), &childDicNodes);
&dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
const int childDicNodesSize = childDicNodes.getSizeAndLock();
for (int i = 0; i < childDicNodesSize; ++i) {
@ -431,7 +431,8 @@ void Suggest::processDicNodeAsDigraph(DicTraverseSession *traverseSession,
void Suggest::processDicNodeAsOmission(
DicTraverseSession *traverseSession, DicNode *dicNode) const {
DicNodeVector childDicNodes;
DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getOffsetDict(), &childDicNodes);
DicNodeUtils::getAllChildDicNodes(
dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
const int size = childDicNodes.getSizeAndLock();
for (int i = 0; i < size; i++) {
@ -456,7 +457,7 @@ void Suggest::processDicNodeAsInsertion(DicTraverseSession *traverseSession,
DicNode *dicNode) const {
const int16_t pointIndex = dicNode->getInputIndex(0);
DicNodeVector childDicNodes;
DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(),
DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes);
const int size = childDicNodes.getSizeAndLock();
for (int i = 0; i < size; i++) {
@ -474,14 +475,14 @@ void Suggest::processDicNodeAsTransposition(DicTraverseSession *traverseSession,
DicNode *dicNode) const {
const int16_t pointIndex = dicNode->getInputIndex(0);
DicNodeVector childDicNodes1;
DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(),
DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1);
const int childSize1 = childDicNodes1.getSizeAndLock();
for (int i = 0; i < childSize1; i++) {
if (childDicNodes1[i]->hasChildren()) {
DicNodeVector childDicNodes2;
DicNodeUtils::getProximityChildDicNodes(
childDicNodes1[i], traverseSession->getOffsetDict(),
childDicNodes1[i], traverseSession->getBinaryDictionaryInfo(),
traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2);
const int childSize2 = childDicNodes2.getSizeAndLock();
for (int j = 0; j < childSize2; j++) {
@ -521,8 +522,8 @@ void Suggest::createNextWordDicNode(DicTraverseSession *traverseSession, DicNode
// Create a non-cached node here.
DicNode newDicNode;
DicNodeUtils::initAsRootWithPreviousWord(traverseSession->getDicRootPos(),
traverseSession->getOffsetDict(), dicNode, &newDicNode);
DicNodeUtils::initAsRootWithPreviousWord(
traverseSession->getBinaryDictionaryInfo(), dicNode, &newDicNode);
const CorrectionType correctionType = spaceSubstitution ?
CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION;
Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode,

2
native/jni/src/suggest/policyimpl/typing/typing_weighting.h
View File

@ -146,7 +146,7 @@ class TypingWeighting : public Weighting {
float getNewWordBigramCost(const DicTraverseSession *const traverseSession,
const DicNode *const dicNode,
MultiBigramMap *const multiBigramMap) const {
return DicNodeUtils::getBigramNodeImprobability(traverseSession->getOffsetDict(),
return DicNodeUtils::getBigramNodeImprobability(traverseSession->getBinaryDictionaryInfo(),
dicNode, multiBigramMap) * ScoringParams::DISTANCE_WEIGHT_LANGUAGE;
}

56
native/jni/src/unigram_dictionary.cpp
View File

@ -19,6 +19,7 @@
#define LOG_TAG "LatinIME: unigram_dictionary.cpp"
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/char_utils.h"
#include "suggest/core/dictionary/dictionary.h"
@ -33,8 +34,9 @@
namespace latinime {
// TODO: check the header
UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags)
: DICT_ROOT(streamStart), ROOT_POS(0),
UnigramDictionary::UnigramDictionary(
const BinaryDictionaryInfo *const binaryDicitonaryInfo, const uint8_t dictFlags)
: mBinaryDicitonaryInfo(binaryDicitonaryInfo),
MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), DICT_FLAGS(dictFlags) {
if (DEBUG_DICT) {
AKLOGI("UnigramDictionary - constructor");
@ -316,9 +318,10 @@ void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
correction->setCorrectionParams(0, 0, 0,
-1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance,
doAutoCompletion, maxErrors);
int rootPosition = ROOT_POS;
int rootPosition = mBinaryDicitonaryInfo->getRootPosition();
// Get the number of children of root, then increment the position
int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition);
int childCount = BinaryFormat::getGroupCountAndForwardPointer(
mBinaryDicitonaryInfo->getDictRoot(), &rootPosition);
int outputIndex = 0;
correction->initCorrectionState(rootPosition, childCount, (inputSize <= 0));
@ -748,7 +751,7 @@ int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, con
int newWord[MAX_WORD_LENGTH];
int depth = 0;
int maxFreq = -1;
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDicitonaryInfo->getDictRoot();
int stackChildCount[MAX_WORD_LENGTH];
int stackInputIndex[MAX_WORD_LENGTH];
int stackSiblingPos[MAX_WORD_LENGTH];
@ -807,7 +810,7 @@ int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, con
}
int UnigramDictionary::getProbability(const int *const inWord, const int length) const {
const uint8_t *const root = DICT_ROOT;
const uint8_t *const root = mBinaryDicitonaryInfo->getDictRoot();
int pos = BinaryFormat::getTerminalPosition(root, inWord, length,
false /* forceLowerCaseSearch */);
if (NOT_VALID_WORD == pos) {
@ -824,7 +827,7 @@ int UnigramDictionary::getProbability(const int *const inWord, const int length)
if (hasMultipleChars) {
pos = BinaryFormat::skipOtherCharacters(root, pos);
} else {
BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos);
BinaryFormat::getCodePointAndForwardPointer(root, &pos);
}
const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos);
return unigramProbability;
@ -866,7 +869,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not.
// - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children)
// - FLAG_HAS_BIGRAMS: whether this node has bigrams or not
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos);
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(
mBinaryDicitonaryInfo->getDictRoot(), &pos);
const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags));
const bool isTerminalNode = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags));
@ -877,7 +881,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// else if FLAG_IS_TERMINAL: the probability
// else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address
// Note that you can't have a node that both is not a terminal and has no children.
int c = BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos);
int c = BinaryFormat::getCodePointAndForwardPointer(
mBinaryDicitonaryInfo->getDictRoot(), &pos);
ASSERT(NOT_A_CODE_POINT != c);
// We are going to loop through each character and make it look like it's a different
@ -891,8 +896,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// We prefetch the next char. If 'c' is the last char of this node, we will have
// NOT_A_CODE_POINT in the next char. From this we can decide whether this virtual node
// should behave as a terminal or not and whether we have children.
const int nextc = hasMultipleChars
? BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CODE_POINT;
const int nextc = hasMultipleChars ? BinaryFormat::getCodePointAndForwardPointer(
mBinaryDicitonaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT;
const bool isLastChar = (NOT_A_CODE_POINT == nextc);
// If there are more chars in this nodes, then this virtual node is not a terminal.
// If we are on the last char, this virtual node is a terminal if this node is.
@ -912,11 +917,11 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// We don't have to output other values because we return false, as in
// "don't traverse children".
if (!isLastChar) {
pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos);
pos = BinaryFormat::skipOtherCharacters(mBinaryDicitonaryInfo->getDictRoot(), pos);
}
pos = BinaryFormat::skipProbability(flags, pos);
*nextSiblingPosition =
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(
mBinaryDicitonaryInfo->getDictRoot(), flags, pos);
return false;
}
@ -929,11 +934,11 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
if (isTerminalNode) {
// The probability should be here, because we come here only if this is actually
// a terminal node, and we are on its last char.
const int unigramProbability =
BinaryFormat::readProbabilityWithoutMovingPointer(DICT_ROOT, pos);
const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(
mBinaryDicitonaryInfo->getDictRoot(), pos);
const int childrenAddressPos = BinaryFormat::skipProbability(flags, pos);
const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos);
TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos);
TerminalAttributes terminalAttributes(mBinaryDicitonaryInfo, flags, attributesPos);
// bigramMap contains the bigram frequencies indexed by addresses for fast lookup.
// bigramFilter is a bloom filter of said frequencies for even faster rejection.
const int probability = ProbabilityUtils::getProbability(
@ -953,16 +958,16 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// remaining char in this group for there can't be any.
if (!hasChildren) {
pos = BinaryFormat::skipProbability(flags, pos);
*nextSiblingPosition =
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(
mBinaryDicitonaryInfo->getDictRoot(), flags, pos);
return false;
}
// Optimization: Prune out words that are too long compared to how much was typed.
if (correction->needsToPrune()) {
pos = BinaryFormat::skipProbability(flags, pos);
*nextSiblingPosition =
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(
mBinaryDicitonaryInfo->getDictRoot(), flags, pos);
if (DEBUG_DICT_FULL) {
AKLOGI("Traversing was pruned.");
}
@ -981,9 +986,12 @@ bool UnigramDictionary::processCurrentNode(const int initialPos,
// Once this is read, we still need to output the number of nodes in the immediate children of
// this node, so we read and output it before returning true, as in "please traverse children".
pos = BinaryFormat::skipProbability(flags, pos);
int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos);
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
*newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos);
int childrenPos = BinaryFormat::readChildrenPosition(
mBinaryDicitonaryInfo->getDictRoot(), flags, pos);
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(
mBinaryDicitonaryInfo->getDictRoot(), flags, pos);
*newCount = BinaryFormat::getGroupCountAndForwardPointer(
mBinaryDicitonaryInfo->getDictRoot(), &childrenPos);
*newChildrenPosition = childrenPos;
return true;
}

10
native/jni/src/unigram_dictionary.h
View File

@ -25,6 +25,7 @@
namespace latinime {
class BinaryDictionaryInfo;
class Correction;
class ProximityInfo;
class TerminalAttributes;
@ -39,7 +40,10 @@ class UnigramDictionary {
static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0;
static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1;
static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2;
UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags);
UnigramDictionary(const BinaryDictionaryInfo *const binaryDicitonaryInfo,
const uint8_t dictFlags);
virtual ~UnigramDictionary();
int getProbability(const int *const inWord, const int length) const;
int getBigramPosition(int pos, int *word, int offset, int length) const;
int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
@ -48,7 +52,6 @@ class UnigramDictionary {
const bool useFullEditDistance, int *outWords, int *frequencies,
int *outputTypes) const;
int getDictFlags() const { return DICT_FLAGS; }
virtual ~UnigramDictionary();
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary);
@ -108,8 +111,7 @@ class UnigramDictionary {
const int outputWordLength, int *freqArray, int *wordLengthArray,
int *outputWord) const;
const uint8_t *const DICT_ROOT;
const int ROOT_POS;
const BinaryDictionaryInfo *const mBinaryDicitonaryInfo;
const int MAX_DIGRAPH_SEARCH_DEPTH;
const int DICT_FLAGS;
};