diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 1b758c680..0631b2255 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -58,6 +58,8 @@ LATIN_IME_CORE_SRC_FILES := \ dic_nodes_cache.cpp) \ $(addprefix suggest/core/dictionary/, \ char_utils.cpp \ + binary_dictionary_format.cpp \ + byte_array_utils.cpp \ dictionary.cpp \ digraph_utils.cpp) \ $(addprefix suggest/core/layout/, \ diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 9f5e2ae73..e80abf6e6 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -35,7 +35,8 @@ #include "jni.h" #include "jni_common.h" #include "suggest_options.h" -#include "suggest/core/dictionary/binary_format.h" +#include "suggest/core/dictionary/binary_dictionary_format.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/dictionary.h" namespace latinime { @@ -110,8 +111,8 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return 0; } Dictionary *dictionary = 0; - if (BinaryFormat::UNKNOWN_FORMAT - == BinaryFormat::detectFormat(static_cast(dictBuf), + if (BinaryDictionaryFormat::UNKNOWN_VERSION + == BinaryDictionaryFormat::detectFormatVersion(static_cast(dictBuf), static_cast(dictSize))) { AKLOGE("DICT: dictionary format is unknown, bad magic number"); #ifdef USE_MMAP_FOR_DICTIONARY @@ -260,7 +261,7 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast(dict); if (!dictionary) return; - const void *dictBuf = dictionary->getDict(); + const void *dictBuf = dictionary->getBinaryDictionaryInfo()->getDictBuf(); if (!dictBuf) return; #ifdef USE_MMAP_FOR_DICTIONARY releaseDictBuf(static_cast(dictBuf) - dictionary->getDictBufAdjust(), diff --git a/native/jni/src/bigram_dictionary.cpp b/native/jni/src/bigram_dictionary.cpp index ebe27994f..1da5212dc 100644 --- a/native/jni/src/bigram_dictionary.cpp +++ b/native/jni/src/bigram_dictionary.cpp @@ -21,6 +21,7 @@ #include "bigram_dictionary.h" #include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" #include "suggest/core/dictionary/bloom_filter.h" #include "suggest/core/dictionary/char_utils.h" @@ -28,7 +29,8 @@ namespace latinime { -BigramDictionary::BigramDictionary(const uint8_t *const streamStart) : DICT_ROOT(streamStart) { +BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo) + : mBinaryDictionaryInfo(binaryDictionaryInfo) { if (DEBUG_DICT) { AKLOGI("BigramDictionary - constructor"); } @@ -103,7 +105,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i // TODO: remove unused arguments, and refrain from storing stuff in members of this class // TODO: have "in" arguments before "out" ones, and make out args explicit in the name - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(prevWord, prevWordLength, false /* forceLowerCaseSearch */); // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams @@ -149,7 +151,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength, const bool forceLowerCaseSearch) const { if (0 >= prevWordLength) return 0; - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength, forceLowerCaseSearch); @@ -170,7 +172,7 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in void BigramDictionary::fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength, std::map *map, uint8_t *filter) const { memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE); - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(prevWord, prevWordLength, false /* forceLowerCaseSearch */); if (0 == pos) { @@ -209,7 +211,7 @@ bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) cons bool BigramDictionary::isValidBigram(const int *word1, int length1, const int *word2, int length2) const { - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(word1, length1, false /* forceLowerCaseSearch */); // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams if (0 == pos) return false; diff --git a/native/jni/src/bigram_dictionary.h b/native/jni/src/bigram_dictionary.h index b86e564c3..8b7a253a2 100644 --- a/native/jni/src/bigram_dictionary.h +++ b/native/jni/src/bigram_dictionary.h @@ -24,9 +24,12 @@ namespace latinime { +class BinaryDictionaryInfo; + class BigramDictionary { public: - BigramDictionary(const uint8_t *const streamStart); + BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo); + int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords, int *frequencies, int *outputTypes) const; void fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength, @@ -35,13 +38,14 @@ class BigramDictionary { ~BigramDictionary(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary); + void addWordBigram(int *word, int length, int probability, int *bigramProbability, int *bigramCodePoints, int *outputTypes) const; bool checkFirstCharacter(int *word, int *inputCodePoints) const; int getBigramListPositionForWord(const int *prevWord, const int prevWordLength, const bool forceLowerCaseSearch) const; - const uint8_t *const DICT_ROOT; + const BinaryDictionaryInfo *const mBinaryDictionaryInfo; // TODO: Re-implement proximity correction for bigram correction static const int MAX_ALTERNATIVES = 1; }; diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp index c754a5ec2..77afa9ee7 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp @@ -20,6 +20,7 @@ #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_utils.h" #include "suggest/core/dicnode/dic_node_vector.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" #include "suggest/core/dictionary/char_utils.h" #include "suggest/core/dictionary/multi_bigram_map.h" @@ -32,20 +33,23 @@ namespace latinime { // Node initialization utils // /////////////////////////////// -/* static */ void DicNodeUtils::initAsRoot(const int rootPos, const uint8_t *const dicRoot, - const int prevWordNodePos, DicNode *newRootNode) { - int curPos = rootPos; +/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo, + const int prevWordNodePos, DicNode *const newRootNode) { + int curPos = binaryDictionaryInfo->getRootPosition(); const int pos = curPos; - const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos); + const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &curPos); const int childrenPos = curPos; newRootNode->initAsRoot(pos, childrenPos, childrenCount, prevWordNodePos); } -/*static */ void DicNodeUtils::initAsRootWithPreviousWord(const int rootPos, - const uint8_t *const dicRoot, DicNode *prevWordLastNode, DicNode *newRootNode) { - int curPos = rootPos; +/*static */ void DicNodeUtils::initAsRootWithPreviousWord( + const BinaryDictionaryInfo *const binaryDictionaryInfo, + DicNode *const prevWordLastNode, DicNode *const newRootNode) { + int curPos = binaryDictionaryInfo->getRootPosition(); const int pos = curPos; - const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos); + const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &curPos); const int childrenPos = curPos; newRootNode->initAsRootWithPreviousWord(prevWordLastNode, pos, childrenPos, childrenCount); } @@ -71,16 +75,19 @@ namespace latinime { } /* static */ int DicNodeUtils::createAndGetLeavingChildNode(DicNode *dicNode, int pos, - const uint8_t *const dicRoot, const int terminalDepth, const ProximityInfoState *pInfoState, - const int pointIndex, const bool exactOnly, const std::vector *const codePointsFilter, - const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, + const std::vector *const codePointsFilter, const ProximityInfo *const pInfo, + DicNodeVector *childDicNodes) { int nextPos = pos; - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dicRoot, &pos); + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos); const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); const bool isTerminal = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags)); const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags); - int codePoint = BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos); + int codePoint = BinaryFormat::getCodePointAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos); ASSERT(NOT_A_CODE_POINT != codePoint); const int nodeCodePoint = codePoint; // TODO: optimize this @@ -90,7 +97,8 @@ namespace latinime { do { const int nextCodePoint = hasMultipleChars - ? BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos) : NOT_A_CODE_POINT; + ? BinaryFormat::getCodePointAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT; const bool isLastChar = (NOT_A_CODE_POINT == nextCodePoint); if (!isLastChar) { additionalWordBuf[additionalSubwordLength++] = nextCodePoint; @@ -98,12 +106,14 @@ namespace latinime { codePoint = nextCodePoint; } while (NOT_A_CODE_POINT != codePoint); - const int probability = - isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(dicRoot, pos) : -1; + const int probability = isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer( + binaryDictionaryInfo->getDictRoot(), pos) : -1; pos = BinaryFormat::skipProbability(flags, pos); - int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(dicRoot, flags, pos) : 0; + int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition( + binaryDictionaryInfo->getDictRoot(), flags, pos) : 0; const int attributesPos = BinaryFormat::skipChildrenPosition(flags, pos); - const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(dicRoot, flags, pos); + const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes( + binaryDictionaryInfo->getDictRoot(), flags, pos); if (isDicNodeFilteredOut(nodeCodePoint, pInfo, codePointsFilter)) { return siblingPos; @@ -111,8 +121,8 @@ namespace latinime { if (!isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, nodeCodePoint)) { return siblingPos; } - const int childrenCount = hasChildren - ? BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &childrenPos) : 0; + const int childrenCount = hasChildren ? BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &childrenPos) : 0; childDicNodes->pushLeavingChild(dicNode, nextPos, flags, childrenPos, attributesPos, siblingPos, nodeCodePoint, childrenCount, probability, -1 /* bigramProbability */, isTerminal, hasMultipleChars, hasChildren, additionalSubwordLength, additionalWordBuf); @@ -148,16 +158,18 @@ namespace latinime { } /* static */ void DicNodeUtils::createAndGetAllLeavingChildNodes(DicNode *dicNode, - const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex, - const bool exactOnly, const std::vector *const codePointsFilter, - const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, + const std::vector *const codePointsFilter, const ProximityInfo *const pInfo, + DicNodeVector *childDicNodes) { const int terminalDepth = dicNode->getLeavingDepth(); const int childCount = dicNode->getChildrenCount(); int nextPos = dicNode->getChildrenPos(); for (int i = 0; i < childCount; i++) { const int filterSize = codePointsFilter ? codePointsFilter->size() : 0; - nextPos = createAndGetLeavingChildNode(dicNode, nextPos, dicRoot, terminalDepth, pInfoState, - pointIndex, exactOnly, codePointsFilter, pInfo, childDicNodes); + nextPos = createAndGetLeavingChildNode(dicNode, nextPos, binaryDictionaryInfo, + terminalDepth, pInfoState, pointIndex, exactOnly, codePointsFilter, pInfo, + childDicNodes); if (!pInfo && filterSize > 0 && childDicNodes->exceeds(filterSize)) { // All code points have been found. break; @@ -165,14 +177,15 @@ namespace latinime { } } -/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, - DicNodeVector *childDicNodes) { - getProximityChildDicNodes(dicNode, dicRoot, 0, 0, false, childDicNodes); +/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) { + getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes); } /* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode, - const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex, - bool exactOnly, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, + const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, + DicNodeVector *childDicNodes) { if (dicNode->isTotalInputSizeExceedingLimit()) { return; } @@ -180,9 +193,9 @@ namespace latinime { DicNodeUtils::createAndGetPassingChildNode(dicNode, pInfoState, pointIndex, exactOnly, childDicNodes); } else { - DicNodeUtils::createAndGetAllLeavingChildNodes(dicNode, dicRoot, pInfoState, pointIndex, - exactOnly, 0 /* codePointsFilter */, 0 /* pInfo */, - childDicNodes); + DicNodeUtils::createAndGetAllLeavingChildNodes( + dicNode, binaryDictionaryInfo, pInfoState, pointIndex, exactOnly, + 0 /* codePointsFilter */, 0 /* pInfo */, childDicNodes); } } @@ -192,19 +205,21 @@ namespace latinime { /** * Computes the combined bigram / unigram cost for the given dicNode. */ -/* static */ float DicNodeUtils::getBigramNodeImprobability(const uint8_t *const dicRoot, +/* static */ float DicNodeUtils::getBigramNodeImprobability( + const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *multiBigramMap) { if (node->isImpossibleBigramWord()) { return static_cast(MAX_VALUE_FOR_WEIGHTING); } - const int probability = getBigramNodeProbability(dicRoot, node, multiBigramMap); + const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap); // TODO: This equation to calculate the improbability looks unreasonable. Investigate this. const float cost = static_cast(MAX_PROBABILITY - probability) / static_cast(MAX_PROBABILITY); return cost; } -/* static */ int DicNodeUtils::getBigramNodeProbability(const uint8_t *const dicRoot, +/* static */ int DicNodeUtils::getBigramNodeProbability( + const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *multiBigramMap) { const int unigramProbability = node->getProbability(); const int wordPos = node->getPos(); @@ -215,9 +230,10 @@ namespace latinime { } if (multiBigramMap) { return multiBigramMap->getBigramProbability( - dicRoot, prevWordPos, wordPos, unigramProbability); + binaryDictionaryInfo, prevWordPos, wordPos, unigramProbability); } - return BinaryFormat::getBigramProbability(dicRoot, prevWordPos, wordPos, unigramProbability); + return BinaryFormat::getBigramProbability( + binaryDictionaryInfo->getDictRoot(), prevWordPos, wordPos, unigramProbability); } /////////////////////////////////////// diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.h b/native/jni/src/suggest/core/dicnode/dic_node_utils.h index 5bc542d05..e198d6181 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.h @@ -24,6 +24,7 @@ namespace latinime { +class BinaryDictionaryInfo; class DicNode; class DicNodeVector; class ProximityInfo; @@ -34,19 +35,20 @@ class DicNodeUtils { public: static int appendTwoWords(const int *src0, const int16_t length0, const int *src1, const int16_t length1, int *dest); - static void initAsRoot(const int rootPos, const uint8_t *const dicRoot, + static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int prevWordNodePos, DicNode *newRootNode); - static void initAsRootWithPreviousWord(const int rootPos, const uint8_t *const dicRoot, + static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNode *prevWordLastNode, DicNode *newRootNode); static void initByCopy(DicNode *srcNode, DicNode *destNode); - static void getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, - DicNodeVector *childDicNodes); - static float getBigramNodeImprobability(const uint8_t *const dicRoot, + static void getAllChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes); + static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *const multiBigramMap); static bool isDicNodeFilteredOut(const int nodeCodePoint, const ProximityInfo *const pInfo, const std::vector *const codePointsFilter); // TODO: Move to private - static void getProximityChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, + static void getProximityChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, DicNodeVector *childDicNodes); @@ -60,16 +62,18 @@ class DicNodeUtils { // Max number of bigrams to look up static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500; - static int getBigramNodeProbability(const uint8_t *const dicRoot, const DicNode *const node, - MultiBigramMap *multiBigramMap); + static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo, + const DicNode *const node, MultiBigramMap *multiBigramMap); static void createAndGetPassingChildNode(DicNode *dicNode, const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, DicNodeVector *childDicNodes); - static void createAndGetAllLeavingChildNodes(DicNode *dicNode, const uint8_t *const dicRoot, + static void createAndGetAllLeavingChildNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, const std::vector *const codePointsFilter, const ProximityInfo *const pInfo, DicNodeVector *childDicNodes); - static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, const uint8_t *const dicRoot, - const int terminalDepth, const ProximityInfoState *pInfoState, const int pointIndex, + static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, const std::vector *const codePointsFilter, const ProximityInfo *const pInfo, DicNodeVector *childDicNodes); diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp new file mode 100644 index 000000000..50e0211d7 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/core/dictionary/binary_dictionary_format.h" + +namespace latinime { + +/** + * Dictionary size + */ +// Any file smaller than this is not a dictionary. +const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; + +/** + * Format versions + */ +// Originally, format version 1 had a 16-bit magic number, then the version number `01' +// then options that must be 0. Hence the first 32-bits of the format are always as follow +// and it's okay to consider them a magic number as a whole. +const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; +const int BinaryDictionaryFormat::FORMAT_VERSION_1_HEADER_SIZE = 5; + +// The versions of Latin IME that only handle format version 1 only test for the magic +// number, so we had to change it so that version 2 files would be rejected by older +// implementations. On this occasion, we made the magic number 32 bits long. +const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; +// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 +const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12; +const int BinaryDictionaryFormat::VERSION_2_MAGIC_NUMBER_SIZE = 4; +const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_VERSION_SIZE = 2; +const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_FLAG_SIZE = 2; + +/* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion( + const uint8_t *const dict, const int dictSize) { + // The magic number is stored big-endian. + // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't + // understand this format. + if (dictSize < DICTIONARY_MINIMUM_SIZE) { + return UNKNOWN_VERSION; + } + const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0); + switch (magicNumber) { + case FORMAT_VERSION_1_MAGIC_NUMBER: + // Format 1 header is exactly 5 bytes long and looks like: + // Magic number (2 bytes) 0x78 0xB1 + // Version number (1 byte) 0x01 + // Options (2 bytes) must be 0x00 0x00 + return VERSION_1; + case FORMAT_VERSION_2_MAGIC_NUMBER: + // Version 2 dictionaries are at least 12 bytes long. + // If this dictionary has the version 2 magic number but is less than 12 bytes long, + // then it's an unknown format and we need to avoid confidently reading the next bytes. + if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) { + return UNKNOWN_VERSION; + } + // Format 2 header is as follows: + // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE + // Version number (2 bytes) 0x00 0x02 + // Options (2 bytes) + // Header size (4 bytes) : integer, big endian + if (ByteArrayUtils::readUint16(dict, 4) == 2) { + return VERSION_2; + } else { + return UNKNOWN_VERSION; + } + default: + return UNKNOWN_VERSION; + } +} + +} // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h new file mode 100644 index 000000000..3aa1662da --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BINARY_DICTIONARY_FORMAT_H +#define LATINIME_BINARY_DICTIONARY_FORMAT_H + +#include + +#include "defines.h" +#include "suggest/core/dictionary/byte_array_utils.h" + +namespace latinime { + +/** + * Methods to handle binary dictionary format version. + * + * Currently, we have a file with a similar name, binary_format.h. binary_format.h contains binary + * reading methods and utility methods for various purposes. + * On the other hand, this file deals with only about dictionary format version. + */ +class BinaryDictionaryFormat { + public: + // TODO: Remove obsolete version logic + enum FORMAT_VERSION { + VERSION_1, + VERSION_2, + UNKNOWN_VERSION + }; + + static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); + + static AK_FORCE_INLINE int getHeaderSize( + const uint8_t *const dict, const FORMAT_VERSION format) { + switch (format) { + case VERSION_1: + return FORMAT_VERSION_1_HEADER_SIZE; + case VERSION_2: + // See the format of the header in the comment in detectFormat() above + return ByteArrayUtils::readUint32(dict, 8); + default: + return S_INT_MAX; + } + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat); + + static const int DICTIONARY_MINIMUM_SIZE; + static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER; + static const int FORMAT_VERSION_1_HEADER_SIZE; + static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER; + static const int FORMAT_VERSION_2_MINIMUM_SIZE; + static const int VERSION_2_MAGIC_NUMBER_SIZE; + static const int VERSION_2_DICTIONARY_VERSION_SIZE ; + static const int VERSION_2_DICTIONARY_FLAG_SIZE; +}; +} // namespace latinime +#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_H */ diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h new file mode 100644 index 000000000..8508c6786 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BINARY_DICTIONARY_INFO_H +#define LATINIME_BINARY_DICTIONARY_INFO_H + +#include + +#include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_format.h" + +namespace latinime { + +class BinaryDictionaryInfo { + public: + BinaryDictionaryInfo(const uint8_t *const dictBuf, const int dictSize) + : mDictBuf(dictBuf), + mFormat(BinaryDictionaryFormat::detectFormatVersion(mDictBuf, dictSize)), + mDictRoot(mDictBuf + BinaryDictionaryFormat::getHeaderSize(mDictBuf, mFormat)) {} + + AK_FORCE_INLINE const uint8_t *getDictBuf() const { + return mDictBuf; + } + + AK_FORCE_INLINE const uint8_t *getDictRoot() const { + return mDictRoot; + } + + AK_FORCE_INLINE BinaryDictionaryFormat::FORMAT_VERSION getFormat() const { + return mFormat; + } + + AK_FORCE_INLINE int getRootPosition() const { + return 0; + } + + private: + DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo); + + const uint8_t *const mDictBuf; + const BinaryDictionaryFormat::FORMAT_VERSION mFormat; + const uint8_t *const mDictRoot; +}; +} +#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */ diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp new file mode 100644 index 000000000..68b1d5d15 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/core/dictionary/byte_array_utils.h" + +namespace latinime { + +const uint8_t ByteArrayUtils::MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; +const uint8_t ByteArrayUtils::CHARACTER_ARRAY_TERMINATOR = 0x1F; + +} // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.h b/native/jni/src/suggest/core/dictionary/byte_array_utils.h new file mode 100644 index 000000000..832b74725 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.h @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BYTE_ARRAY_UTILS_H +#define LATINIME_BYTE_ARRAY_UTILS_H + +#include + +#include "defines.h" + +namespace latinime { + +/** + * Utility methods for reading byte arrays. + */ +class ByteArrayUtils { + public: + /** + * Integer + * + * Each method read a corresponding size integer in a big endian manner. + */ + static AK_FORCE_INLINE uint32_t readUint32(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 24) ^ (buffer[pos + 1] << 16) + ^ (buffer[pos + 2] << 8) ^ buffer[pos + 3]; + } + + static AK_FORCE_INLINE uint32_t readUint24(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 16) ^ (buffer[pos + 1] << 8) ^ buffer[pos + 2]; + } + + static AK_FORCE_INLINE uint16_t readUint16(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 8) ^ buffer[pos + 1]; + } + + static AK_FORCE_INLINE uint8_t readUint8(const uint8_t *const buffer, const int pos) { + return buffer[pos]; + } + + static AK_FORCE_INLINE uint32_t readUint32andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint32_t value = readUint32(buffer, *pos); + *pos += 4; + return value; + } + + static AK_FORCE_INLINE uint32_t readUint24andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint32_t value = readUint24(buffer, *pos); + *pos += 3; + return value; + } + + static AK_FORCE_INLINE uint16_t readUint16andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint16_t value = readUint16(buffer, *pos); + *pos += 2; + return value; + } + + static AK_FORCE_INLINE uint8_t readUint8andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + return buffer[(*pos)++]; + } + + /** + * Code Point + * + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + */ + static AK_FORCE_INLINE int readCodePoint(const uint8_t *const buffer, const int pos) { + int p = pos; + return readCodePointAndAdvancePosition(buffer, &p); + } + + static AK_FORCE_INLINE int readCodePointAndAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint8_t firstByte = readUint8(buffer, *pos); + if (firstByte < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + if (firstByte == CHARACTER_ARRAY_TERMINATOR) { + *pos += 1; + return NOT_A_CODE_POINT; + } else { + return readUint24andAdvancePosition(buffer, pos); + } + } else { + *pos += 1; + return firstByte; + } + } + + /** + * String (array of code points) + * + * Reads code points until the terminator is found. + */ + // Returns the length of the string. + static int readStringAndAdvancePosition(const uint8_t *const buffer, int *const pos, + int *const outBuffer, const int maxLength) { + int length = 0; + int codePoint = readCodePointAndAdvancePosition(buffer, pos); + while (NOT_A_CODE_POINT != codePoint && length < maxLength) { + outBuffer[length++] = codePoint; + codePoint = readCodePointAndAdvancePosition(buffer, pos); + } + return length; + } + + // Advances the position and returns the length of the string. + static int advancePositionToBehindString( + const uint8_t *const buffer, int *const pos, const int maxLength) { + int length = 0; + int codePoint = readCodePointAndAdvancePosition(buffer, pos); + while (NOT_A_CODE_POINT != codePoint && length < maxLength) { + codePoint = readCodePointAndAdvancePosition(buffer, pos); + } + return length; + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils); + + static const uint8_t MINIMAL_ONE_BYTE_CHARACTER_VALUE; + static const uint8_t CHARACTER_ARRAY_TERMINATOR; +}; +} // namespace latinime +#endif /* LATINIME_BYTE_ARRAY_UTILS_H */ diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index 1939c7420..53a6751d2 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -34,13 +34,11 @@ namespace latinime { Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust) - : mDict(static_cast(dict)), - mOffsetDict((static_cast(dict)) - + BinaryFormat::getHeaderSize(mDict, dictSize)), + : mBinaryDicitonaryInfo(static_cast(dict), dictSize), mDictSize(dictSize), mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust), - mUnigramDictionary(new UnigramDictionary(mOffsetDict, - BinaryFormat::getFlags(mDict, dictSize))), - mBigramDictionary(new BigramDictionary(mOffsetDict)), + mUnigramDictionary(new UnigramDictionary(&mBinaryDicitonaryInfo, + BinaryFormat::getFlags(mBinaryDicitonaryInfo.getDictBuf(), dictSize))), + mBigramDictionary(new BigramDictionary(&mBinaryDicitonaryInfo)), mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())), mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) { } diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index e6861a3dd..771837bc6 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -20,6 +20,7 @@ #include #include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" namespace latinime { @@ -64,11 +65,8 @@ class Dictionary { int getProbability(const int *word, int length) const; bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const; - const uint8_t *getDict() const { // required to release dictionary buffer - return mDict; - } - const uint8_t *getOffsetDict() const { - return mOffsetDict; + const BinaryDictionaryInfo *getBinaryDictionaryInfo() const { + return &mBinaryDicitonaryInfo; } int getDictSize() const { return mDictSize; } int getMmapFd() const { return mMmapFd; } @@ -78,9 +76,8 @@ class Dictionary { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); - const uint8_t *mDict; - const uint8_t *mOffsetDict; + const BinaryDictionaryInfo mBinaryDicitonaryInfo; // Used only for the mmap version of dictionary loading, but we use these as dummy variables // also for the malloc version. const int mDictSize; diff --git a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h index fcac98f35..b17c75803 100644 --- a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h +++ b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h @@ -17,10 +17,9 @@ #ifndef LATINIME_MULTI_BIGRAM_MAP_H #define LATINIME_MULTI_BIGRAM_MAP_H -#include - #include "defines.h" #include "hash_map_compat.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" namespace latinime { @@ -35,20 +34,20 @@ class MultiBigramMap { // Look up the bigram probability for the given word pair from the cached bigram maps. // Also caches the bigrams if there is space remaining and they have not been cached already. - int getBigramProbability(const uint8_t *const dicRoot, const int wordPosition, - const int nextWordPosition, const int unigramProbability) { + int getBigramProbability(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const int wordPosition, const int nextWordPosition, const int unigramProbability) { hash_map_compat::const_iterator mapPosition = mBigramMaps.find(wordPosition); if (mapPosition != mBigramMaps.end()) { return mapPosition->second.getBigramProbability(nextWordPosition, unigramProbability); } if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) { - addBigramsForWordPosition(dicRoot, wordPosition); + addBigramsForWordPosition(binaryDicitonaryInfo, wordPosition); return mBigramMaps[wordPosition].getBigramProbability( nextWordPosition, unigramProbability); } - return BinaryFormat::getBigramProbability( - dicRoot, wordPosition, nextWordPosition, unigramProbability); + return BinaryFormat::getBigramProbability(binaryDicitonaryInfo->getDictRoot(), + wordPosition, nextWordPosition, unigramProbability); } void clear() { @@ -63,8 +62,9 @@ class MultiBigramMap { BigramMap() : mBigramMap(DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP) {} ~BigramMap() {} - void init(const uint8_t *const dicRoot, int position) { - BinaryFormat::fillBigramProbabilityToHashMap(dicRoot, position, &mBigramMap); + void init(const BinaryDictionaryInfo *const binaryDicitonaryInfo, const int position) { + BinaryFormat::fillBigramProbabilityToHashMap( + binaryDicitonaryInfo->getDictRoot(), position, &mBigramMap); } inline int getBigramProbability(const int nextWordPosition, const int unigramProbability) @@ -78,8 +78,9 @@ class MultiBigramMap { hash_map_compat mBigramMap; }; - void addBigramsForWordPosition(const uint8_t *const dicRoot, const int position) { - mBigramMaps[position].init(dicRoot, position); + void addBigramsForWordPosition(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const int position) { + mBigramMaps[position].init(binaryDicitonaryInfo, position); } hash_map_compat mBigramMaps; diff --git a/native/jni/src/suggest/core/dictionary/terminal_attributes.h b/native/jni/src/suggest/core/dictionary/terminal_attributes.h index 8377c603d..bbd9af090 100644 --- a/native/jni/src/suggest/core/dictionary/terminal_attributes.h +++ b/native/jni/src/suggest/core/dictionary/terminal_attributes.h @@ -19,6 +19,7 @@ #include +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" namespace latinime { @@ -32,8 +33,9 @@ class TerminalAttributes { public: class ShortcutIterator { public: - ShortcutIterator(const uint8_t *dict, const int pos, const uint8_t flags) - : mDict(dict), mPos(pos), + ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int pos, + const uint8_t flags) + : mBinaryDicitionaryInfo(binaryDictionaryInfo), mPos(pos), mHasNextShortcutTarget(0 != (flags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS)) { } @@ -44,11 +46,13 @@ class TerminalAttributes { // Gets the shortcut target itself as an int string. For parameters and return value // see BinaryFormat::getWordAtAddress. inline int getNextShortcutTarget(const int maxDepth, int *outWord, int *outFreq) { - const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos); + const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer( + mBinaryDicitionaryInfo->getDictRoot(), &mPos); mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT); unsigned int i; for (i = 0; i < MAX_WORD_LENGTH; ++i) { - const int codePoint = BinaryFormat::getCodePointAndForwardPointer(mDict, &mPos); + const int codePoint = BinaryFormat::getCodePointAndForwardPointer( + mBinaryDicitionaryInfo->getDictRoot(), &mPos); if (NOT_A_CODE_POINT == codePoint) break; outWord[i] = codePoint; } @@ -57,19 +61,21 @@ class TerminalAttributes { } private: - const uint8_t *const mDict; + const BinaryDictionaryInfo *const mBinaryDicitionaryInfo; int mPos; bool mHasNextShortcutTarget; }; - TerminalAttributes(const uint8_t *const dict, const uint8_t flags, const int pos) - : mDict(dict), mFlags(flags), mStartPos(pos) { + TerminalAttributes(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const uint8_t flags, const int pos) + : mBinaryDicitionaryInfo(binaryDicitonaryInfo), mFlags(flags), mStartPos(pos) { } inline ShortcutIterator getShortcutIterator() const { // The size of the shortcuts is stored here so that the whole shortcut chunk can be // skipped quickly, so we ignore it. - return ShortcutIterator(mDict, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags); + return ShortcutIterator( + mBinaryDicitionaryInfo, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags); } bool isBlacklistedOrNotAWord() const { @@ -78,7 +84,7 @@ class TerminalAttributes { private: DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes); - const uint8_t *const mDict; + const BinaryDictionaryInfo *const mBinaryDicitionaryInfo; const uint8_t mFlags; const int mStartPos; }; diff --git a/native/jni/src/suggest/core/policy/weighting.cpp b/native/jni/src/suggest/core/policy/weighting.cpp index 7ad568e25..0c57ca001 100644 --- a/native/jni/src/suggest/core/policy/weighting.cpp +++ b/native/jni/src/suggest/core/policy/weighting.cpp @@ -142,7 +142,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n case CT_TERMINAL: { const float languageImprobability = DicNodeUtils::getBigramNodeImprobability( - traverseSession->getOffsetDict(), dicNode, multiBigramMap); + traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap); return weighting->getTerminalLanguageCost(traverseSession, dicNode, languageImprobability); } case CT_NEW_WORD_SPACE_SUBSTITUTION: diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp index 4e634500c..be293df42 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp +++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp @@ -20,6 +20,7 @@ #include "dic_traverse_wrapper.h" #include "jni.h" #include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" #include "suggest/core/dictionary/dictionary.h" @@ -65,7 +66,8 @@ static TraverseSessionFactoryRegisterer traverseSessionFactoryRegisterer; void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord, int prevWordLength, const SuggestOptions *const suggestOptions) { mDictionary = dictionary; - mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(mDictionary->getDict(), + mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier( + mDictionary->getBinaryDictionaryInfo()->getDictBuf(), mDictionary->getDictSize()); mSuggestOptions = suggestOptions; if (!prevWord) { @@ -73,12 +75,14 @@ void DicTraverseSession::init(const Dictionary *const dictionary, const int *pre return; } // TODO: merge following similar calls to getTerminalPosition into one case-insensitive call. - mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord, + mPrevWordPos = BinaryFormat::getTerminalPosition( + dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord, prevWordLength, false /* forceLowerCaseSearch */); if (mPrevWordPos == NOT_VALID_WORD) { // Check bigrams for lower-cased previous word if original was not found. Useful for // auto-capitalized words like "The [current_word]". - mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord, + mPrevWordPos = BinaryFormat::getTerminalPosition( + dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord, prevWordLength, true /* forceLowerCaseSearch */); } } @@ -93,8 +97,8 @@ void DicTraverseSession::setupForGetSuggestions(const ProximityInfo *pInfo, maxSpatialDistance, maxPointerCount); } -const uint8_t *DicTraverseSession::getOffsetDict() const { - return mDictionary->getOffsetDict(); +const BinaryDictionaryInfo *DicTraverseSession::getBinaryDictionaryInfo() const { + return mDictionary->getBinaryDictionaryInfo(); } int DicTraverseSession::getDictFlags() const { diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h index e5c7f8e0c..3b6a3dc8c 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.h +++ b/native/jni/src/suggest/core/session/dic_traverse_session.h @@ -28,6 +28,7 @@ namespace latinime { +class BinaryDictionaryInfo; class Dictionary; class ProximityInfo; class SuggestOptions; @@ -56,7 +57,7 @@ class DicTraverseSession { void resetCache(const int nextActiveCacheSize, const int maxWords); // TODO: Remove - const uint8_t *getOffsetDict() const; + const BinaryDictionaryInfo *getBinaryDictionaryInfo() const; int getDictFlags() const; //-------------------- diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index 720222363..1f108e400 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -105,8 +105,8 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession, int commitPo traverseSession->resetCache(TRAVERSAL->getMaxCacheSize(), MAX_RESULTS); // Create a new dic node here DicNode rootNode; - DicNodeUtils::initAsRoot(traverseSession->getDicRootPos(), - traverseSession->getOffsetDict(), traverseSession->getPrevWordPos(), &rootNode); + DicNodeUtils::initAsRoot(traverseSession->getBinaryDictionaryInfo(), + traverseSession->getPrevWordPos(), &rootNode); traverseSession->getDicTraverseCache()->copyPushActive(&rootNode); } } @@ -158,7 +158,7 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen terminalIndex, doubleLetterTerminalIndex, doubleLetterLevel); const float compoundDistance = terminalDicNode->getCompoundDistance(languageWeight) + doubleLetterCost; - const TerminalAttributes terminalAttributes(traverseSession->getOffsetDict(), + const TerminalAttributes terminalAttributes(traverseSession->getBinaryDictionaryInfo(), terminalDicNode->getFlags(), terminalDicNode->getAttributesPos()); const bool isPossiblyOffensiveWord = terminalDicNode->getProbability() <= 0; const bool isExactMatch = terminalDicNode->isExactMatch(); @@ -284,7 +284,7 @@ void Suggest::expandCurrentDicNodes(DicTraverseSession *traverseSession) const { } DicNodeUtils::getAllChildDicNodes( - &dicNode, traverseSession->getOffsetDict(), &childDicNodes); + &dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes); const int childDicNodesSize = childDicNodes.getSizeAndLock(); for (int i = 0; i < childDicNodesSize; ++i) { @@ -431,7 +431,8 @@ void Suggest::processDicNodeAsDigraph(DicTraverseSession *traverseSession, void Suggest::processDicNodeAsOmission( DicTraverseSession *traverseSession, DicNode *dicNode) const { DicNodeVector childDicNodes; - DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getOffsetDict(), &childDicNodes); + DicNodeUtils::getAllChildDicNodes( + dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes); const int size = childDicNodes.getSizeAndLock(); for (int i = 0; i < size; i++) { @@ -456,7 +457,7 @@ void Suggest::processDicNodeAsInsertion(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes; - DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(), + DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes); const int size = childDicNodes.getSizeAndLock(); for (int i = 0; i < size; i++) { @@ -474,14 +475,14 @@ void Suggest::processDicNodeAsTransposition(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes1; - DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(), + DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1); const int childSize1 = childDicNodes1.getSizeAndLock(); for (int i = 0; i < childSize1; i++) { if (childDicNodes1[i]->hasChildren()) { DicNodeVector childDicNodes2; DicNodeUtils::getProximityChildDicNodes( - childDicNodes1[i], traverseSession->getOffsetDict(), + childDicNodes1[i], traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2); const int childSize2 = childDicNodes2.getSizeAndLock(); for (int j = 0; j < childSize2; j++) { @@ -521,8 +522,8 @@ void Suggest::createNextWordDicNode(DicTraverseSession *traverseSession, DicNode // Create a non-cached node here. DicNode newDicNode; - DicNodeUtils::initAsRootWithPreviousWord(traverseSession->getDicRootPos(), - traverseSession->getOffsetDict(), dicNode, &newDicNode); + DicNodeUtils::initAsRootWithPreviousWord( + traverseSession->getBinaryDictionaryInfo(), dicNode, &newDicNode); const CorrectionType correctionType = spaceSubstitution ? CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION; Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode, diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h index 9e0c0d2f7..36d496a27 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h +++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h @@ -146,7 +146,7 @@ class TypingWeighting : public Weighting { float getNewWordBigramCost(const DicTraverseSession *const traverseSession, const DicNode *const dicNode, MultiBigramMap *const multiBigramMap) const { - return DicNodeUtils::getBigramNodeImprobability(traverseSession->getOffsetDict(), + return DicNodeUtils::getBigramNodeImprobability(traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap) * ScoringParams::DISTANCE_WEIGHT_LANGUAGE; } diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp index 66a8b8542..fa31538db 100644 --- a/native/jni/src/unigram_dictionary.cpp +++ b/native/jni/src/unigram_dictionary.cpp @@ -19,6 +19,7 @@ #define LOG_TAG "LatinIME: unigram_dictionary.cpp" #include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/binary_format.h" #include "suggest/core/dictionary/char_utils.h" #include "suggest/core/dictionary/dictionary.h" @@ -32,8 +33,9 @@ namespace latinime { // TODO: check the header -UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags) - : DICT_ROOT(streamStart), ROOT_POS(0), +UnigramDictionary::UnigramDictionary( + const BinaryDictionaryInfo *const binaryDicitonaryInfo, const uint8_t dictFlags) + : mBinaryDicitonaryInfo(binaryDicitonaryInfo), MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), DICT_FLAGS(dictFlags) { if (DEBUG_DICT) { AKLOGI("UnigramDictionary - constructor"); @@ -315,9 +317,10 @@ void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, correction->setCorrectionParams(0, 0, 0, -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance, doAutoCompletion, maxErrors); - int rootPosition = ROOT_POS; + int rootPosition = mBinaryDicitonaryInfo->getRootPosition(); // Get the number of children of root, then increment the position - int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition); + int childCount = BinaryFormat::getGroupCountAndForwardPointer( + mBinaryDicitonaryInfo->getDictRoot(), &rootPosition); int outputIndex = 0; correction->initCorrectionState(rootPosition, childCount, (inputSize <= 0)); @@ -747,7 +750,7 @@ int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, con int newWord[MAX_WORD_LENGTH]; int depth = 0; int maxFreq = -1; - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDicitonaryInfo->getDictRoot(); int stackChildCount[MAX_WORD_LENGTH]; int stackInputIndex[MAX_WORD_LENGTH]; int stackSiblingPos[MAX_WORD_LENGTH]; @@ -806,7 +809,7 @@ int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, con } int UnigramDictionary::getProbability(const int *const inWord, const int length) const { - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDicitonaryInfo->getDictRoot(); int pos = BinaryFormat::getTerminalPosition(root, inWord, length, false /* forceLowerCaseSearch */); if (NOT_VALID_WORD == pos) { @@ -823,7 +826,7 @@ int UnigramDictionary::getProbability(const int *const inWord, const int length) if (hasMultipleChars) { pos = BinaryFormat::skipOtherCharacters(root, pos); } else { - BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos); + BinaryFormat::getCodePointAndForwardPointer(root, &pos); } const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos); return unigramProbability; @@ -865,7 +868,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not. // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children) // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos); + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer( + mBinaryDicitonaryInfo->getDictRoot(), &pos); const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); const bool isTerminalNode = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags)); @@ -876,7 +880,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // else if FLAG_IS_TERMINAL: the probability // else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address // Note that you can't have a node that both is not a terminal and has no children. - int c = BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos); + int c = BinaryFormat::getCodePointAndForwardPointer( + mBinaryDicitonaryInfo->getDictRoot(), &pos); ASSERT(NOT_A_CODE_POINT != c); // We are going to loop through each character and make it look like it's a different @@ -890,8 +895,8 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // We prefetch the next char. If 'c' is the last char of this node, we will have // NOT_A_CODE_POINT in the next char. From this we can decide whether this virtual node // should behave as a terminal or not and whether we have children. - const int nextc = hasMultipleChars - ? BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CODE_POINT; + const int nextc = hasMultipleChars ? BinaryFormat::getCodePointAndForwardPointer( + mBinaryDicitonaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT; const bool isLastChar = (NOT_A_CODE_POINT == nextc); // If there are more chars in this nodes, then this virtual node is not a terminal. // If we are on the last char, this virtual node is a terminal if this node is. @@ -911,11 +916,11 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // We don't have to output other values because we return false, as in // "don't traverse children". if (!isLastChar) { - pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos); + pos = BinaryFormat::skipOtherCharacters(mBinaryDicitonaryInfo->getDictRoot(), pos); } pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes( + mBinaryDicitonaryInfo->getDictRoot(), flags, pos); return false; } @@ -928,11 +933,11 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, if (isTerminalNode) { // The probability should be here, because we come here only if this is actually // a terminal node, and we are on its last char. - const int unigramProbability = - BinaryFormat::readProbabilityWithoutMovingPointer(DICT_ROOT, pos); + const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer( + mBinaryDicitonaryInfo->getDictRoot(), pos); const int childrenAddressPos = BinaryFormat::skipProbability(flags, pos); const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos); - TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos); + TerminalAttributes terminalAttributes(mBinaryDicitonaryInfo, flags, attributesPos); // bigramMap contains the bigram frequencies indexed by addresses for fast lookup. // bigramFilter is a bloom filter of said frequencies for even faster rejection. const int probability = BinaryFormat::getProbability(initialPos, bigramMap, bigramFilter, @@ -952,16 +957,16 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // remaining char in this group for there can't be any. if (!hasChildren) { pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes( + mBinaryDicitonaryInfo->getDictRoot(), flags, pos); return false; } // Optimization: Prune out words that are too long compared to how much was typed. if (correction->needsToPrune()) { pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes( + mBinaryDicitonaryInfo->getDictRoot(), flags, pos); if (DEBUG_DICT_FULL) { AKLOGI("Traversing was pruned."); } @@ -980,9 +985,12 @@ bool UnigramDictionary::processCurrentNode(const int initialPos, // Once this is read, we still need to output the number of nodes in the immediate children of // this node, so we read and output it before returning true, as in "please traverse children". pos = BinaryFormat::skipProbability(flags, pos); - int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos); - *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); - *newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos); + int childrenPos = BinaryFormat::readChildrenPosition( + mBinaryDicitonaryInfo->getDictRoot(), flags, pos); + *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes( + mBinaryDicitonaryInfo->getDictRoot(), flags, pos); + *newCount = BinaryFormat::getGroupCountAndForwardPointer( + mBinaryDicitonaryInfo->getDictRoot(), &childrenPos); *newChildrenPosition = childrenPos; return true; } diff --git a/native/jni/src/unigram_dictionary.h b/native/jni/src/unigram_dictionary.h index a50503256..4edd1f847 100644 --- a/native/jni/src/unigram_dictionary.h +++ b/native/jni/src/unigram_dictionary.h @@ -25,6 +25,7 @@ namespace latinime { +class BinaryDictionaryInfo; class Correction; class ProximityInfo; class TerminalAttributes; @@ -39,7 +40,10 @@ class UnigramDictionary { static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0; static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1; static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2; - UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags); + + UnigramDictionary(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const uint8_t dictFlags); + virtual ~UnigramDictionary(); int getProbability(const int *const inWord, const int length) const; int getBigramPosition(int pos, int *word, int offset, int length) const; int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, @@ -48,7 +52,6 @@ class UnigramDictionary { const bool useFullEditDistance, int *outWords, int *frequencies, int *outputTypes) const; int getDictFlags() const { return DICT_FLAGS; } - virtual ~UnigramDictionary(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary); @@ -108,8 +111,7 @@ class UnigramDictionary { const int outputWordLength, int *freqArray, int *wordLengthArray, int *outputWord) const; - const uint8_t *const DICT_ROOT; - const int ROOT_POS; + const BinaryDictionaryInfo *const mBinaryDicitonaryInfo; const int MAX_DIGRAPH_SEARCH_DEPTH; const int DICT_FLAGS; };