Unbundle members in unigram_dictionary

Change-Id: Id737d943d20e3de3db568162caf40d3e956c7fae
This commit is contained in:
satok 2011-12-14 15:04:58 +09:00
parent 2c4b21d7ca
commit 1147c7bac9
5 changed files with 113 additions and 101 deletions

View file

@ -198,7 +198,7 @@ static void dumpWord(const unsigned short* word, const int length) {
#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f #define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
#define HALF_SCORE_SQUARED_RADIUS 32.0f #define HALF_SCORE_SQUARED_RADIUS 32.0f
// This should be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java // This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
// This is only used for the size of array. Not to be used in c functions. // This is only used for the size of array. Not to be used in c functions.
#define MAX_WORD_LENGTH_INTERNAL 48 #define MAX_WORD_LENGTH_INTERNAL 48

View file

@ -38,6 +38,8 @@ Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust,
LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF)); LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF));
} }
} }
mCorrection = new Correction(typedLetterMultiplier, fullWordMultiplier);
mWordsPriorityQueue = new WordsPriorityQueue(maxWords, maxWordLength);
mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier, mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier,
maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION); maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION);
mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives, mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives,
@ -45,6 +47,8 @@ Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust,
} }
Dictionary::~Dictionary() { Dictionary::~Dictionary() {
delete mCorrection;
delete mWordsPriorityQueue;
delete mUnigramDictionary; delete mUnigramDictionary;
delete mBigramDictionary; delete mBigramDictionary;
} }

View file

@ -19,9 +19,11 @@
#include "bigram_dictionary.h" #include "bigram_dictionary.h"
#include "char_utils.h" #include "char_utils.h"
#include "correction.h"
#include "defines.h" #include "defines.h"
#include "proximity_info.h" #include "proximity_info.h"
#include "unigram_dictionary.h" #include "unigram_dictionary.h"
#include "words_priority_queue.h"
namespace latinime { namespace latinime {
@ -29,9 +31,11 @@ class Dictionary {
public: public:
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler, Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler,
int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives); int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates, int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates,
int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) { int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) {
return mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, return mUnigramDictionary->getSuggestions(proximityInfo, mWordsPriorityQueue,
mCorrection, xcoordinates, ycoordinates, codes,
codesSize, flags, outWords, frequencies); codesSize, flags, outWords, frequencies);
} }
@ -77,6 +81,8 @@ private:
const bool IS_LATEST_DICT_VERSION; const bool IS_LATEST_DICT_VERSION;
UnigramDictionary *mUnigramDictionary; UnigramDictionary *mUnigramDictionary;
BigramDictionary *mBigramDictionary; BigramDictionary *mBigramDictionary;
WordsPriorityQueue *mWordsPriorityQueue;
Correction *mCorrection;
}; };
// public static utility methods // public static utility methods

View file

@ -48,21 +48,23 @@ UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typed
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("UnigramDictionary - constructor"); LOGI("UnigramDictionary - constructor");
} }
mCorrection = new Correction(typedLetterMultiplier, fullWordMultiplier);
mWordsPriorityQueue = new WordsPriorityQueue(maxWords, maxWordLength);
} }
UnigramDictionary::~UnigramDictionary() { UnigramDictionary::~UnigramDictionary() {
delete mCorrection;
delete mWordsPriorityQueue;
} }
static inline unsigned int getCodesBufferSize(const int* codes, const int codesSize, static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize,
const int MAX_PROXIMITY_CHARS) { const int MAX_PROXIMITY_CHARS) {
return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize; return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize;
} }
bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codesSize) const { // TODO: This needs to take an const unsigned short* and not tinker with its contents
static inline void addWord(
unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) {
queue->push(frequency, word, length);
}
bool UnigramDictionary::isDigraph(const int *codes, const int i, const int codesSize) const {
// There can't be a digraph if we don't have at least 2 characters to examine // There can't be a digraph if we don't have at least 2 characters to examine
if (i + 2 > codesSize) return false; if (i + 2 > codesSize) return false;
@ -88,9 +90,10 @@ bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codes
// codesSrc is the current point in the user-input, original, content-unmodified buffer. // codesSrc is the current point in the user-input, original, content-unmodified buffer.
// codesRemain is the remaining size in codesSrc. // codesRemain is the remaining size in codesSrc.
void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, const int *xcoordinates, const int *ycoordinates, const int *codesBuffer,
const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain, const int codesBufferSize, const int flags, const int *codesSrc,
const int currentDepth, int* codesDest) { const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
WordsPriorityQueue *queue) {
if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) { if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) {
for (int i = 0; i < codesRemain; ++i) { for (int i = 0; i < codesRemain; ++i) {
@ -107,7 +110,7 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
codesBuffer, codesBufferSize, flags, codesBuffer, codesBufferSize, flags,
codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1, codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1,
currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS); currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, correction, queue);
// Copy the second char of the digraph in place, then continue processing on // Copy the second char of the digraph in place, then continue processing on
// the remaining part of the word. // the remaining part of the word.
@ -115,8 +118,9 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS, memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS,
BYTES_IN_ONE_CHAR); BYTES_IN_ONE_CHAR);
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
codesBuffer, codesBufferSize, flags, codesSrc + i * MAX_PROXIMITY_CHARS, codesBuffer, codesBufferSize, flags,
codesRemain - i, currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS); codesSrc + i * MAX_PROXIMITY_CHARS, codesRemain - i, currentDepth + 1,
codesDest + i * MAX_PROXIMITY_CHARS, correction, queue);
return; return;
} }
} }
@ -132,25 +136,28 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
memcpy(codesDest, codesSrc, remainingBytes); memcpy(codesDest, codesSrc, remainingBytes);
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer, getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
(codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, flags); (codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, flags, correction,
queue);
} }
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueue *queue,
const int *ycoordinates, const int *codes, const int codesSize, const int flags, Correction *correction, const int *xcoordinates, const int *ycoordinates, const int *codes,
unsigned short *outWords, int *frequencies) { const int codesSize, const int flags, unsigned short *outWords, int *frequencies) {
WordsPriorityQueue* masterQueue = queue;
Correction* masterCorrection = correction;
if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags) if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags)
{ // Incrementally tune the word and try all possibilities { // Incrementally tune the word and try all possibilities
int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)]; int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)];
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
codesSize, flags, codes, codesSize, 0, codesBuffer); codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, masterQueue);
} else { // Normal processing } else { // Normal processing
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags); getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags,
masterCorrection, masterQueue);
} }
PROF_START(20); PROF_START(20);
const int suggestedWordsCount = const int suggestedWordsCount = masterQueue->outputSuggestions(frequencies, outWords);
mWordsPriorityQueue->outputSuggestions(frequencies, outWords);
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("Returning %d words", suggestedWordsCount); LOGI("Returning %d words", suggestedWordsCount);
@ -170,23 +177,22 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *x
} }
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize, const int *xcoordinates, const int *ycoordinates, const int *codes,
const int flags) { const int inputLength, const int flags, Correction *correction, WordsPriorityQueue *queue) {
PROF_OPEN; PROF_OPEN;
PROF_START(0); PROF_START(0);
initSuggestions( initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, queue);
proximityInfo, xcoordinates, ycoordinates, codes, codesSize); if (DEBUG_DICT) assert(codesSize == inputLength);
if (DEBUG_DICT) assert(codesSize == mInputLength);
const int maxDepth = min(mInputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
mCorrection->initCorrection(mProximityInfo, mInputLength, maxDepth); correction->initCorrection(proximityInfo, inputLength, maxDepth);
PROF_END(0); PROF_END(0);
const bool useFullEditDistance = USE_FULL_EDIT_DISTANCE & flags; const bool useFullEditDistance = USE_FULL_EDIT_DISTANCE & flags;
// TODO: remove // TODO: remove
PROF_START(1); PROF_START(1);
getSuggestionCandidates(useFullEditDistance); getSuggestionCandidates(useFullEditDistance, inputLength, correction, queue);
PROF_END(1); PROF_END(1);
PROF_START(2); PROF_START(2);
@ -204,12 +210,13 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
PROF_START(5); PROF_START(5);
// Suggestions with missing space // Suggestions with missing space
if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER
&& mInputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) { && inputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) {
for (int i = 1; i < codesSize; ++i) { for (int i = 1; i < inputLength; ++i) {
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("--- Suggest missing space characters %d", i); LOGI("--- Suggest missing space characters %d", i);
} }
getMissingSpaceWords(mInputLength, i, mCorrection, useFullEditDistance); getMissingSpaceWords(
inputLength, i, proximityInfo, correction, useFullEditDistance, queue);
} }
} }
PROF_END(5); PROF_END(5);
@ -217,7 +224,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
PROF_START(6); PROF_START(6);
if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) { if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) {
// The first and last "mistyped spaces" are taken care of by excessive character handling // The first and last "mistyped spaces" are taken care of by excessive character handling
for (int i = 1; i < codesSize - 1; ++i) { for (int i = 1; i < inputLength - 1; ++i) {
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("--- Suggest words with proximity space %d", i); LOGI("--- Suggest words with proximity space %d", i);
} }
@ -228,7 +235,8 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
i, x, y, proximityInfo->hasSpaceProximity(x, y)); i, x, y, proximityInfo->hasSpaceProximity(x, y));
} }
if (proximityInfo->hasSpaceProximity(x, y)) { if (proximityInfo->hasSpaceProximity(x, y)) {
getMistypedSpaceWords(mInputLength, i, mCorrection, useFullEditDistance); getMistypedSpaceWords(
inputLength, i, proximityInfo, correction, useFullEditDistance, queue);
} }
} }
} }
@ -236,93 +244,83 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
} }
void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates, void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
const int *yCoordinates, const int *codes, const int codesSize) { const int *yCoordinates, const int *codes, const int codesSize,
WordsPriorityQueue *queue) {
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("initSuggest"); LOGI("initSuggest");
} }
mInputLength = codesSize;
proximityInfo->setInputParams(codes, codesSize, xCoordinates, yCoordinates); proximityInfo->setInputParams(codes, codesSize, xCoordinates, yCoordinates);
mProximityInfo = proximityInfo; queue->clear();
mWordsPriorityQueue->clear();
}
// TODO: This needs to take an const unsigned short* and not tinker with its contents
void UnigramDictionary::addWord(unsigned short *word, int length, int frequency) {
mWordsPriorityQueue->push(frequency, word, length);
} }
static const char QUOTE = '\''; static const char QUOTE = '\'';
static const char SPACE = ' '; static const char SPACE = ' ';
void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance) { void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
const int inputLength, Correction *correction, WordsPriorityQueue *queue) {
// TODO: Remove setCorrectionParams // TODO: Remove setCorrectionParams
mCorrection->setCorrectionParams(0, 0, 0, correction->setCorrectionParams(0, 0, 0,
-1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance); -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance);
int rootPosition = ROOT_POS; int rootPosition = ROOT_POS;
// Get the number of children of root, then increment the position // Get the number of children of root, then increment the position
int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition); int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
int outputIndex = 0; int outputIndex = 0;
mCorrection->initCorrectionState(rootPosition, childCount, (mInputLength <= 0)); correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0));
// Depth first search // Depth first search
while (outputIndex >= 0) { while (outputIndex >= 0) {
if (mCorrection->initProcessState(outputIndex)) { if (correction->initProcessState(outputIndex)) {
int siblingPos = mCorrection->getTreeSiblingPos(outputIndex); int siblingPos = correction->getTreeSiblingPos(outputIndex);
int firstChildPos; int firstChildPos;
const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
mCorrection, &childCount, &firstChildPos, &siblingPos); correction, &childCount, &firstChildPos, &siblingPos, queue);
// Update next sibling pos // Update next sibling pos
mCorrection->setTreeSiblingPos(outputIndex, siblingPos); correction->setTreeSiblingPos(outputIndex, siblingPos);
if (needsToTraverseChildrenNodes) { if (needsToTraverseChildrenNodes) {
// Goes to child node // Goes to child node
outputIndex = mCorrection->goDownTree(outputIndex, childCount, firstChildPos); outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos);
} }
} else { } else {
// Goes to parent sibling node // Goes to parent sibling node
outputIndex = mCorrection->getTreeParentIndex(outputIndex); outputIndex = correction->getTreeParentIndex(outputIndex);
} }
} }
} }
void UnigramDictionary::getMissingSpaceWords( void UnigramDictionary::getMissingSpaceWords(
const int inputLength, const int missingSpacePos, Correction *correction, const int inputLength, const int missingSpacePos, ProximityInfo *proximityInfo,
const bool useFullEditDistance) { Correction *correction, const bool useFullEditDistance, WordsPriorityQueue *queue) {
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */, correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
-1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos, -1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos,
useFullEditDistance); useFullEditDistance);
getSplitTwoWordsSuggestion(inputLength, correction); getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queue);
} }
void UnigramDictionary::getMistypedSpaceWords( void UnigramDictionary::getMistypedSpaceWords(
const int inputLength, const int spaceProximityPos, Correction *correction, const int inputLength, const int spaceProximityPos, ProximityInfo *proximityInfo,
const bool useFullEditDistance) { Correction *correction, const bool useFullEditDistance, WordsPriorityQueue *queue) {
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */, correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
-1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */, -1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */,
useFullEditDistance); useFullEditDistance);
getSplitTwoWordsSuggestion(inputLength, correction); getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queue);
} }
inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c, inline void UnigramDictionary::onTerminal(
const int inputIndex, const int skipPos, const int depth) { const int freq, Correction *correction, WordsPriorityQueue *queue) {
const unsigned short userTypedChar = mProximityInfo->getPrimaryCharAt(inputIndex);
// Skip the ' or other letter and continue deeper
return (c == QUOTE && userTypedChar != QUOTE) || skipPos == depth;
}
inline void UnigramDictionary::onTerminal(const int freq, Correction *correction) {
int wordLength; int wordLength;
unsigned short* wordPointer; unsigned short* wordPointer;
const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength); const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength);
if (finalFreq >= 0) { if (finalFreq >= 0) {
addWord(wordPointer, wordLength, finalFreq); addWord(wordPointer, wordLength, finalFreq, queue);
} }
} }
void UnigramDictionary::getSplitTwoWordsSuggestion( void UnigramDictionary::getSplitTwoWordsSuggestion(
const int inputLength, Correction* correction) { const int inputLength, ProximityInfo *proximityInfo, Correction *correction,
WordsPriorityQueue *queue) {
const int spaceProximityPos = correction->getSpaceProximityPos(); const int spaceProximityPos = correction->getSpaceProximityPos();
const int missingSpacePos = correction->getMissingSpacePos(); const int missingSpacePos = correction->getMissingSpacePos();
if (DEBUG_DICT) { if (DEBUG_DICT) {
@ -347,7 +345,8 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
const int newWordLength = firstWordLength + secondWordLength + 1; const int newWordLength = firstWordLength + secondWordLength + 1;
// Allocating variable length array on stack // Allocating variable length array on stack
unsigned short word[newWordLength]; unsigned short word[newWordLength];
const int firstFreq = getMostFrequentWordLike(firstWordStartPos, firstWordLength, mWord); const int firstFreq = getMostFrequentWordLike(
firstWordStartPos, firstWordLength, proximityInfo, mWord);
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("First freq: %d", firstFreq); LOGI("First freq: %d", firstFreq);
} }
@ -357,7 +356,8 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
word[i] = mWord[i]; word[i] = mWord[i];
} }
const int secondFreq = getMostFrequentWordLike(secondWordStartPos, secondWordLength, mWord); const int secondFreq = getMostFrequentWordLike(
secondWordStartPos, secondWordLength, proximityInfo, mWord);
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("Second freq: %d", secondFreq); LOGI("Second freq: %d", secondFreq);
} }
@ -368,22 +368,22 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
word[i] = mWord[i - firstWordLength - 1]; word[i] = mWord[i - firstWordLength - 1];
} }
const int pairFreq = mCorrection->getFreqForSplitTwoWords(firstFreq, secondFreq, word); const int pairFreq = correction->getFreqForSplitTwoWords(firstFreq, secondFreq, word);
if (DEBUG_DICT) { if (DEBUG_DICT) {
LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength); LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength);
} }
addWord(word, newWordLength, pairFreq); addWord(word, newWordLength, pairFreq, queue);
return; return;
} }
// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous // Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
// interface. // interface.
inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex, inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex,
const int inputLength, unsigned short *word) { const int inputLength, ProximityInfo *proximityInfo, unsigned short *word) {
uint16_t inWord[inputLength]; uint16_t inWord[inputLength];
for (int i = 0; i < inputLength; ++i) { for (int i = 0; i < inputLength; ++i) {
inWord[i] = (uint16_t)mProximityInfo->getPrimaryCharAt(startInputIndex + i); inWord[i] = (uint16_t)proximityInfo->getPrimaryCharAt(startInputIndex + i);
} }
return getMostFrequentWordLikeInner(inWord, inputLength, word); return getMostFrequentWordLikeInner(inWord, inputLength, word);
} }
@ -534,7 +534,7 @@ int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offs
// given level, as output into newCount when traversing this level's parent. // given level, as output into newCount when traversing this level's parent.
inline bool UnigramDictionary::processCurrentNode(const int initialPos, inline bool UnigramDictionary::processCurrentNode(const int initialPos,
Correction *correction, int *newCount, Correction *correction, int *newCount,
int *newChildrenPosition, int *nextSiblingPosition) { int *newChildrenPosition, int *nextSiblingPosition, WordsPriorityQueue *queue) {
if (DEBUG_DICT) { if (DEBUG_DICT) {
correction->checkState(); correction->checkState();
} }
@ -613,7 +613,7 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos,
// The frequency should be here, because we come here only if this is actually // The frequency should be here, because we come here only if this is actually
// a terminal node, and we are on its last char. // a terminal node, and we are on its last char.
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos); const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
onTerminal(freq, mCorrection); onTerminal(freq, correction, queue);
} }
// If there are more chars in this node, then this virtual node has children. // If there are more chars in this node, then this virtual node has children.

View file

@ -66,7 +66,8 @@ public:
const bool isLatestDictVersion); const bool isLatestDictVersion);
bool isValidWord(const uint16_t* const inWord, const int length) const; bool isValidWord(const uint16_t* const inWord, const int length) const;
int getBigramPosition(int pos, unsigned short *word, int offset, int length) const; int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueue *queue,
Correction *correction, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize, const int flags, const int *ycoordinates, const int *codes, const int codesSize, const int flags,
unsigned short *outWords, int *frequencies); unsigned short *outWords, int *frequencies);
virtual ~UnigramDictionary(); virtual ~UnigramDictionary();
@ -74,32 +75,38 @@ public:
private: private:
void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize, const int flags); const int *ycoordinates, const int *codes, const int inputLength,
bool isDigraph(const int* codes, const int i, const int codesSize) const; const int flags, Correction *correction, WordsPriorityQueue *queue);
bool isDigraph(const int *codes, const int i, const int codesSize) const;
void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain, const int codesBufferSize, const int flags, const int* codesSrc,
const int currentDepth, int* codesDest); const int codesRemain, const int currentDepth, int* codesDest, Correction *correction,
WordsPriorityQueue* queue);
void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize); const int *ycoordinates, const int *codes, const int codesSize,
void getSuggestionCandidates(const bool useFullEditDistance); WordsPriorityQueue *queue);
void addWord(unsigned short *word, int length, int frequency); void getSuggestionCandidates(
void getSplitTwoWordsSuggestion(const int inputLength, Correction *correction); const bool useFullEditDistance, const int inputLength, Correction *correction,
WordsPriorityQueue* queue);
void getSplitTwoWordsSuggestion(const int inputLength, ProximityInfo *proximityInfo,
Correction *correction, WordsPriorityQueue *queue);
void getMissingSpaceWords(const int inputLength, const int missingSpacePos, void getMissingSpaceWords(const int inputLength, const int missingSpacePos,
Correction *correction, const bool useFullEditDistance); ProximityInfo *proximityInfo, Correction *correction,
const bool useFullEditDistance, WordsPriorityQueue *queue);
void getMistypedSpaceWords(const int inputLength, const int spaceProximityPos, void getMistypedSpaceWords(const int inputLength, const int spaceProximityPos,
Correction *correction, const bool useFullEditDistance); ProximityInfo *proximityInfo, Correction *correction,
void onTerminal(const int freq, Correction *correction); const bool useFullEditDistance, WordsPriorityQueue *queue);
void onTerminal(const int freq, Correction *correction, WordsPriorityQueue *queue);
bool needsToSkipCurrentNode(const unsigned short c, bool needsToSkipCurrentNode(const unsigned short c,
const int inputIndex, const int skipPos, const int depth); const int inputIndex, const int skipPos, const int depth);
// Process a node by considering proximity, missing and excessive character // Process a node by considering proximity, missing and excessive character
bool processCurrentNode(const int initialPos, bool processCurrentNode(const int initialPos, Correction *correction, int *newCount,
Correction *correction, int *newCount, int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueue *queue);
int *newChildPosition, int *nextSiblingPosition);
int getMostFrequentWordLike(const int startInputIndex, const int inputLength, int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
unsigned short *word); ProximityInfo *proximityInfo, unsigned short *word);
int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length, int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
short unsigned int* outWord); short unsigned int *outWord);
const uint8_t* const DICT_ROOT; const uint8_t* const DICT_ROOT;
const int MAX_WORD_LENGTH; const int MAX_WORD_LENGTH;
@ -122,13 +129,8 @@ private:
}; };
static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[]; static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
WordsPriorityQueue *mWordsPriorityQueue; // Still bundled members
ProximityInfo *mProximityInfo; unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
Correction *mCorrection;
int mInputLength;
// MAX_WORD_LENGTH_INTERNAL must be bigger than MAX_WORD_LENGTH
unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];
int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove