From 8124e64dccb2027fcdcf19b76490769087f55cc2 Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Thu, 16 Jun 2011 22:33:41 +0900 Subject: [PATCH] New dict format, step 2 Move some methods around and make static some methods Bug: 4392433 Change-Id: I2bbe98aec118a416d21d1e293638e1d324505b9b --- native/src/bigram_dictionary.cpp | 3 +- native/src/dictionary.cpp | 39 +------------------------ native/src/dictionary.h | 1 - native/src/unigram_dictionary.cpp | 48 +++++++++++++++++++++++++++++-- native/src/unigram_dictionary.h | 4 +-- 5 files changed, 49 insertions(+), 46 deletions(-) diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp index 36761b88d..4bc436537 100644 --- a/native/src/bigram_dictionary.cpp +++ b/native/src/bigram_dictionary.cpp @@ -111,8 +111,7 @@ int BigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, i mMaxBigrams = maxBigrams; if (HAS_BIGRAM && IS_LATEST_DICT_VERSION) { - int pos = mParentDictionary->isValidWordRec( - DICTIONARY_HEADER_SIZE, prevWord, 0, prevWordLength); + int pos = mParentDictionary->isValidWord(prevWord, prevWordLength); if (DEBUG_DICT) { LOGI("Pos -> %d", pos); } diff --git a/native/src/dictionary.cpp b/native/src/dictionary.cpp index d9ef8f3b7..a49769bdb 100644 --- a/native/src/dictionary.cpp +++ b/native/src/dictionary.cpp @@ -53,45 +53,8 @@ bool Dictionary::hasBigram() { return ((mDict[1] & 0xFF) == 1); } -// TODO: use uint32_t instead of unsigned short bool Dictionary::isValidWord(unsigned short *word, int length) { - if (IS_LATEST_DICT_VERSION) { - return (isValidWordRec(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD); - } else { - return (isValidWordRec(0, word, 0, length) != NOT_VALID_WORD); - } + return mUnigramDictionary->isValidWord(word, length); } -int Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) { - // returns address of bigram data of that word - // return -99 if not found - - int count = Dictionary::getCount(mDict, &pos); - unsigned short currentChar = (unsigned short) word[offset]; - for (int j = 0; j < count; j++) { - unsigned short c = Dictionary::getChar(mDict, &pos); - int terminal = Dictionary::getTerminal(mDict, &pos); - int childPos = Dictionary::getAddress(mDict, &pos); - if (c == currentChar) { - if (offset == length - 1) { - if (terminal) { - return (pos+1); - } - } else { - if (childPos != 0) { - int t = isValidWordRec(childPos, word, offset + 1, length); - if (t > 0) { - return t; - } - } - } - } - if (terminal) { - Dictionary::getFreq(mDict, IS_LATEST_DICT_VERSION, &pos); - } - // There could be two instances of each alphabet - upper and lower case. So continue - // looking ... - } - return NOT_VALID_WORD; -} } // namespace latinime diff --git a/native/src/dictionary.h b/native/src/dictionary.h index 13b2a2816..3b360d93a 100644 --- a/native/src/dictionary.h +++ b/native/src/dictionary.h @@ -43,7 +43,6 @@ public: } bool isValidWord(unsigned short *word, int length); - int isValidWordRec(int pos, unsigned short *word, int offset, int length); void *getDict() { return (void *)mDict; } int getDictSize() { return mDictSize; } int getMmapFd() { return mMmapFd; } diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp index cb8f5076c..91e3c8134 100644 --- a/native/src/unigram_dictionary.cpp +++ b/native/src/unigram_dictionary.cpp @@ -265,8 +265,7 @@ void UnigramDictionary::initSuggestions(const int *codes, const int codesSize, mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2; } -void UnigramDictionary::registerNextLetter( - unsigned short c, int *nextLetters, int nextLettersSize) { +static inline void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize) { if (c < nextLettersSize) { nextLetters[c]++; } @@ -322,7 +321,7 @@ bool UnigramDictionary::addWord(unsigned short *word, int length, int frequency) return false; } -unsigned short UnigramDictionary::toBaseLowerCase(unsigned short c) { +static inline unsigned short toBaseLowerCase(unsigned short c) { if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) { c = BASE_CHARS[c]; } @@ -924,4 +923,47 @@ inline bool UnigramDictionary::processCurrentNodeForExactMatch(const int firstCh return false; } } + +// TODO: use uint32_t instead of unsigned short +bool UnigramDictionary::isValidWord(unsigned short *word, int length) { + if (IS_LATEST_DICT_VERSION) { + return (isValidWordRec(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD); + } else { + return (isValidWordRec(0, word, 0, length) != NOT_VALID_WORD); + } +} + +int UnigramDictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) { + // returns address of bigram data of that word + // return -99 if not found + + int count = Dictionary::getCount(DICT_ROOT, &pos); + unsigned short currentChar = (unsigned short) word[offset]; + for (int j = 0; j < count; j++) { + unsigned short c = Dictionary::getChar(DICT_ROOT, &pos); + int terminal = Dictionary::getTerminal(DICT_ROOT, &pos); + int childPos = Dictionary::getAddress(DICT_ROOT, &pos); + if (c == currentChar) { + if (offset == length - 1) { + if (terminal) { + return (pos+1); + } + } else { + if (childPos != 0) { + int t = isValidWordRec(childPos, word, offset + 1, length); + if (t > 0) { + return t; + } + } + } + } + if (terminal) { + Dictionary::getFreq(DICT_ROOT, IS_LATEST_DICT_VERSION, &pos); + } + // There could be two instances of each alphabet - upper and lower case. So continue + // looking ... + } + return NOT_VALID_WORD; +} + } // namespace latinime diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h index b72ba95ea..cf871fffe 100644 --- a/native/src/unigram_dictionary.h +++ b/native/src/unigram_dictionary.h @@ -39,6 +39,7 @@ public: UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler, int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, const bool isLatestDictVersion); + bool isValidWord(unsigned short *word, int length); int getSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize, const int flags, unsigned short *outWords, int *frequencies); @@ -58,6 +59,7 @@ private: void getSuggestionCandidates(const int skipPos, const int excessivePos, const int transposedPos, int *nextLetters, const int nextLettersSize, const int maxDepth); + int isValidWordRec(int pos, unsigned short *word, int offset, int length); void getVersionNumber(); bool checkIfDictVersionIsLatest(); int getAddress(int *pos); @@ -65,7 +67,6 @@ private: int wideStrLen(unsigned short *str); bool sameAsTyped(unsigned short *word, int length); bool addWord(unsigned short *word, int length, int frequency); - unsigned short toBaseLowerCase(unsigned short c); void getWordsRec(const int childrenCount, const int pos, const int depth, const int maxDepth, const bool traverseAllNodes, const int snr, const int inputIndex, const int diffs, const int skipPos, const int excessivePos, const int transposedPos, int *nextLetters, @@ -79,7 +80,6 @@ private: void getWordsOld(const int initialPos, const int inputLength, const int skipPos, const int excessivePos, const int transposedPos, int *nextLetters, const int nextLettersSize); - void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize); int calculateFinalFreq(const int inputIndex, const int depth, const int snr, const int skipPos, const int excessivePos, const int transposedPos, const int freq, const bool sameLength) const;