Merge "Unbundle members in unigram_dictionary"
This commit is contained in:
commit
c115aed98f
5 changed files with 113 additions and 101 deletions
|
@ -198,7 +198,7 @@ static void dumpWord(const unsigned short* word, const int length) {
|
||||||
#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
|
#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f
|
||||||
#define HALF_SCORE_SQUARED_RADIUS 32.0f
|
#define HALF_SCORE_SQUARED_RADIUS 32.0f
|
||||||
|
|
||||||
// This should be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
|
// This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
|
||||||
// This is only used for the size of array. Not to be used in c functions.
|
// This is only used for the size of array. Not to be used in c functions.
|
||||||
#define MAX_WORD_LENGTH_INTERNAL 48
|
#define MAX_WORD_LENGTH_INTERNAL 48
|
||||||
|
|
||||||
|
|
|
@ -38,6 +38,8 @@ Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust,
|
||||||
LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF));
|
LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
mCorrection = new Correction(typedLetterMultiplier, fullWordMultiplier);
|
||||||
|
mWordsPriorityQueue = new WordsPriorityQueue(maxWords, maxWordLength);
|
||||||
mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier,
|
mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier,
|
||||||
maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION);
|
maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION);
|
||||||
mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives,
|
mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives,
|
||||||
|
@ -45,6 +47,8 @@ Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust,
|
||||||
}
|
}
|
||||||
|
|
||||||
Dictionary::~Dictionary() {
|
Dictionary::~Dictionary() {
|
||||||
|
delete mCorrection;
|
||||||
|
delete mWordsPriorityQueue;
|
||||||
delete mUnigramDictionary;
|
delete mUnigramDictionary;
|
||||||
delete mBigramDictionary;
|
delete mBigramDictionary;
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,9 +19,11 @@
|
||||||
|
|
||||||
#include "bigram_dictionary.h"
|
#include "bigram_dictionary.h"
|
||||||
#include "char_utils.h"
|
#include "char_utils.h"
|
||||||
|
#include "correction.h"
|
||||||
#include "defines.h"
|
#include "defines.h"
|
||||||
#include "proximity_info.h"
|
#include "proximity_info.h"
|
||||||
#include "unigram_dictionary.h"
|
#include "unigram_dictionary.h"
|
||||||
|
#include "words_priority_queue.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
@ -29,9 +31,11 @@ class Dictionary {
|
||||||
public:
|
public:
|
||||||
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler,
|
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler,
|
||||||
int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
|
int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
|
||||||
|
|
||||||
int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates,
|
int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates,
|
||||||
int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) {
|
int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) {
|
||||||
return mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
|
return mUnigramDictionary->getSuggestions(proximityInfo, mWordsPriorityQueue,
|
||||||
|
mCorrection, xcoordinates, ycoordinates, codes,
|
||||||
codesSize, flags, outWords, frequencies);
|
codesSize, flags, outWords, frequencies);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -77,6 +81,8 @@ private:
|
||||||
const bool IS_LATEST_DICT_VERSION;
|
const bool IS_LATEST_DICT_VERSION;
|
||||||
UnigramDictionary *mUnigramDictionary;
|
UnigramDictionary *mUnigramDictionary;
|
||||||
BigramDictionary *mBigramDictionary;
|
BigramDictionary *mBigramDictionary;
|
||||||
|
WordsPriorityQueue *mWordsPriorityQueue;
|
||||||
|
Correction *mCorrection;
|
||||||
};
|
};
|
||||||
|
|
||||||
// public static utility methods
|
// public static utility methods
|
||||||
|
|
|
@ -48,21 +48,23 @@ UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typed
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("UnigramDictionary - constructor");
|
LOGI("UnigramDictionary - constructor");
|
||||||
}
|
}
|
||||||
mCorrection = new Correction(typedLetterMultiplier, fullWordMultiplier);
|
|
||||||
mWordsPriorityQueue = new WordsPriorityQueue(maxWords, maxWordLength);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
UnigramDictionary::~UnigramDictionary() {
|
UnigramDictionary::~UnigramDictionary() {
|
||||||
delete mCorrection;
|
|
||||||
delete mWordsPriorityQueue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline unsigned int getCodesBufferSize(const int* codes, const int codesSize,
|
static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize,
|
||||||
const int MAX_PROXIMITY_CHARS) {
|
const int MAX_PROXIMITY_CHARS) {
|
||||||
return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize;
|
return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codesSize) const {
|
// TODO: This needs to take an const unsigned short* and not tinker with its contents
|
||||||
|
static inline void addWord(
|
||||||
|
unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) {
|
||||||
|
queue->push(frequency, word, length);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool UnigramDictionary::isDigraph(const int *codes, const int i, const int codesSize) const {
|
||||||
|
|
||||||
// There can't be a digraph if we don't have at least 2 characters to examine
|
// There can't be a digraph if we don't have at least 2 characters to examine
|
||||||
if (i + 2 > codesSize) return false;
|
if (i + 2 > codesSize) return false;
|
||||||
|
@ -88,9 +90,10 @@ bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codes
|
||||||
// codesSrc is the current point in the user-input, original, content-unmodified buffer.
|
// codesSrc is the current point in the user-input, original, content-unmodified buffer.
|
||||||
// codesRemain is the remaining size in codesSrc.
|
// codesRemain is the remaining size in codesSrc.
|
||||||
void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
|
void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
|
||||||
const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
|
const int *xcoordinates, const int *ycoordinates, const int *codesBuffer,
|
||||||
const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain,
|
const int codesBufferSize, const int flags, const int *codesSrc,
|
||||||
const int currentDepth, int* codesDest) {
|
const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
|
||||||
|
WordsPriorityQueue *queue) {
|
||||||
|
|
||||||
if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) {
|
if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) {
|
||||||
for (int i = 0; i < codesRemain; ++i) {
|
for (int i = 0; i < codesRemain; ++i) {
|
||||||
|
@ -107,7 +110,7 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
|
||||||
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
||||||
codesBuffer, codesBufferSize, flags,
|
codesBuffer, codesBufferSize, flags,
|
||||||
codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1,
|
codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1,
|
||||||
currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS);
|
currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, correction, queue);
|
||||||
|
|
||||||
// Copy the second char of the digraph in place, then continue processing on
|
// Copy the second char of the digraph in place, then continue processing on
|
||||||
// the remaining part of the word.
|
// the remaining part of the word.
|
||||||
|
@ -115,8 +118,9 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
|
||||||
memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS,
|
memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS,
|
||||||
BYTES_IN_ONE_CHAR);
|
BYTES_IN_ONE_CHAR);
|
||||||
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
||||||
codesBuffer, codesBufferSize, flags, codesSrc + i * MAX_PROXIMITY_CHARS,
|
codesBuffer, codesBufferSize, flags,
|
||||||
codesRemain - i, currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS);
|
codesSrc + i * MAX_PROXIMITY_CHARS, codesRemain - i, currentDepth + 1,
|
||||||
|
codesDest + i * MAX_PROXIMITY_CHARS, correction, queue);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -132,25 +136,28 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
|
||||||
memcpy(codesDest, codesSrc, remainingBytes);
|
memcpy(codesDest, codesSrc, remainingBytes);
|
||||||
|
|
||||||
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
||||||
(codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, flags);
|
(codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, flags, correction,
|
||||||
|
queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueue *queue,
|
||||||
const int *ycoordinates, const int *codes, const int codesSize, const int flags,
|
Correction *correction, const int *xcoordinates, const int *ycoordinates, const int *codes,
|
||||||
unsigned short *outWords, int *frequencies) {
|
const int codesSize, const int flags, unsigned short *outWords, int *frequencies) {
|
||||||
|
|
||||||
|
WordsPriorityQueue* masterQueue = queue;
|
||||||
|
Correction* masterCorrection = correction;
|
||||||
if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags)
|
if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags)
|
||||||
{ // Incrementally tune the word and try all possibilities
|
{ // Incrementally tune the word and try all possibilities
|
||||||
int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)];
|
int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)];
|
||||||
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
||||||
codesSize, flags, codes, codesSize, 0, codesBuffer);
|
codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, masterQueue);
|
||||||
} else { // Normal processing
|
} else { // Normal processing
|
||||||
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags);
|
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags,
|
||||||
|
masterCorrection, masterQueue);
|
||||||
}
|
}
|
||||||
|
|
||||||
PROF_START(20);
|
PROF_START(20);
|
||||||
const int suggestedWordsCount =
|
const int suggestedWordsCount = masterQueue->outputSuggestions(frequencies, outWords);
|
||||||
mWordsPriorityQueue->outputSuggestions(frequencies, outWords);
|
|
||||||
|
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("Returning %d words", suggestedWordsCount);
|
LOGI("Returning %d words", suggestedWordsCount);
|
||||||
|
@ -170,23 +177,22 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *x
|
||||||
}
|
}
|
||||||
|
|
||||||
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
||||||
const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize,
|
const int *xcoordinates, const int *ycoordinates, const int *codes,
|
||||||
const int flags) {
|
const int inputLength, const int flags, Correction *correction, WordsPriorityQueue *queue) {
|
||||||
|
|
||||||
PROF_OPEN;
|
PROF_OPEN;
|
||||||
PROF_START(0);
|
PROF_START(0);
|
||||||
initSuggestions(
|
initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, queue);
|
||||||
proximityInfo, xcoordinates, ycoordinates, codes, codesSize);
|
if (DEBUG_DICT) assert(codesSize == inputLength);
|
||||||
if (DEBUG_DICT) assert(codesSize == mInputLength);
|
|
||||||
|
|
||||||
const int maxDepth = min(mInputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
|
const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
|
||||||
mCorrection->initCorrection(mProximityInfo, mInputLength, maxDepth);
|
correction->initCorrection(proximityInfo, inputLength, maxDepth);
|
||||||
PROF_END(0);
|
PROF_END(0);
|
||||||
|
|
||||||
const bool useFullEditDistance = USE_FULL_EDIT_DISTANCE & flags;
|
const bool useFullEditDistance = USE_FULL_EDIT_DISTANCE & flags;
|
||||||
// TODO: remove
|
// TODO: remove
|
||||||
PROF_START(1);
|
PROF_START(1);
|
||||||
getSuggestionCandidates(useFullEditDistance);
|
getSuggestionCandidates(useFullEditDistance, inputLength, correction, queue);
|
||||||
PROF_END(1);
|
PROF_END(1);
|
||||||
|
|
||||||
PROF_START(2);
|
PROF_START(2);
|
||||||
|
@ -204,12 +210,13 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
||||||
PROF_START(5);
|
PROF_START(5);
|
||||||
// Suggestions with missing space
|
// Suggestions with missing space
|
||||||
if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER
|
if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER
|
||||||
&& mInputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) {
|
&& inputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) {
|
||||||
for (int i = 1; i < codesSize; ++i) {
|
for (int i = 1; i < inputLength; ++i) {
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("--- Suggest missing space characters %d", i);
|
LOGI("--- Suggest missing space characters %d", i);
|
||||||
}
|
}
|
||||||
getMissingSpaceWords(mInputLength, i, mCorrection, useFullEditDistance);
|
getMissingSpaceWords(
|
||||||
|
inputLength, i, proximityInfo, correction, useFullEditDistance, queue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
PROF_END(5);
|
PROF_END(5);
|
||||||
|
@ -217,7 +224,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
||||||
PROF_START(6);
|
PROF_START(6);
|
||||||
if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) {
|
if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) {
|
||||||
// The first and last "mistyped spaces" are taken care of by excessive character handling
|
// The first and last "mistyped spaces" are taken care of by excessive character handling
|
||||||
for (int i = 1; i < codesSize - 1; ++i) {
|
for (int i = 1; i < inputLength - 1; ++i) {
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("--- Suggest words with proximity space %d", i);
|
LOGI("--- Suggest words with proximity space %d", i);
|
||||||
}
|
}
|
||||||
|
@ -228,7 +235,8 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
||||||
i, x, y, proximityInfo->hasSpaceProximity(x, y));
|
i, x, y, proximityInfo->hasSpaceProximity(x, y));
|
||||||
}
|
}
|
||||||
if (proximityInfo->hasSpaceProximity(x, y)) {
|
if (proximityInfo->hasSpaceProximity(x, y)) {
|
||||||
getMistypedSpaceWords(mInputLength, i, mCorrection, useFullEditDistance);
|
getMistypedSpaceWords(
|
||||||
|
inputLength, i, proximityInfo, correction, useFullEditDistance, queue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -236,93 +244,83 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
|
void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
|
||||||
const int *yCoordinates, const int *codes, const int codesSize) {
|
const int *yCoordinates, const int *codes, const int codesSize,
|
||||||
|
WordsPriorityQueue *queue) {
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("initSuggest");
|
LOGI("initSuggest");
|
||||||
}
|
}
|
||||||
mInputLength = codesSize;
|
|
||||||
proximityInfo->setInputParams(codes, codesSize, xCoordinates, yCoordinates);
|
proximityInfo->setInputParams(codes, codesSize, xCoordinates, yCoordinates);
|
||||||
mProximityInfo = proximityInfo;
|
queue->clear();
|
||||||
mWordsPriorityQueue->clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: This needs to take an const unsigned short* and not tinker with its contents
|
|
||||||
void UnigramDictionary::addWord(unsigned short *word, int length, int frequency) {
|
|
||||||
mWordsPriorityQueue->push(frequency, word, length);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static const char QUOTE = '\'';
|
static const char QUOTE = '\'';
|
||||||
static const char SPACE = ' ';
|
static const char SPACE = ' ';
|
||||||
|
|
||||||
void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance) {
|
void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
|
||||||
|
const int inputLength, Correction *correction, WordsPriorityQueue *queue) {
|
||||||
// TODO: Remove setCorrectionParams
|
// TODO: Remove setCorrectionParams
|
||||||
mCorrection->setCorrectionParams(0, 0, 0,
|
correction->setCorrectionParams(0, 0, 0,
|
||||||
-1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance);
|
-1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance);
|
||||||
int rootPosition = ROOT_POS;
|
int rootPosition = ROOT_POS;
|
||||||
// Get the number of children of root, then increment the position
|
// Get the number of children of root, then increment the position
|
||||||
int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
|
int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
|
||||||
int outputIndex = 0;
|
int outputIndex = 0;
|
||||||
|
|
||||||
mCorrection->initCorrectionState(rootPosition, childCount, (mInputLength <= 0));
|
correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0));
|
||||||
|
|
||||||
// Depth first search
|
// Depth first search
|
||||||
while (outputIndex >= 0) {
|
while (outputIndex >= 0) {
|
||||||
if (mCorrection->initProcessState(outputIndex)) {
|
if (correction->initProcessState(outputIndex)) {
|
||||||
int siblingPos = mCorrection->getTreeSiblingPos(outputIndex);
|
int siblingPos = correction->getTreeSiblingPos(outputIndex);
|
||||||
int firstChildPos;
|
int firstChildPos;
|
||||||
|
|
||||||
const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
|
const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
|
||||||
mCorrection, &childCount, &firstChildPos, &siblingPos);
|
correction, &childCount, &firstChildPos, &siblingPos, queue);
|
||||||
// Update next sibling pos
|
// Update next sibling pos
|
||||||
mCorrection->setTreeSiblingPos(outputIndex, siblingPos);
|
correction->setTreeSiblingPos(outputIndex, siblingPos);
|
||||||
|
|
||||||
if (needsToTraverseChildrenNodes) {
|
if (needsToTraverseChildrenNodes) {
|
||||||
// Goes to child node
|
// Goes to child node
|
||||||
outputIndex = mCorrection->goDownTree(outputIndex, childCount, firstChildPos);
|
outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Goes to parent sibling node
|
// Goes to parent sibling node
|
||||||
outputIndex = mCorrection->getTreeParentIndex(outputIndex);
|
outputIndex = correction->getTreeParentIndex(outputIndex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UnigramDictionary::getMissingSpaceWords(
|
void UnigramDictionary::getMissingSpaceWords(
|
||||||
const int inputLength, const int missingSpacePos, Correction *correction,
|
const int inputLength, const int missingSpacePos, ProximityInfo *proximityInfo,
|
||||||
const bool useFullEditDistance) {
|
Correction *correction, const bool useFullEditDistance, WordsPriorityQueue *queue) {
|
||||||
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
||||||
-1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos,
|
-1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos,
|
||||||
useFullEditDistance);
|
useFullEditDistance);
|
||||||
getSplitTwoWordsSuggestion(inputLength, correction);
|
getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
void UnigramDictionary::getMistypedSpaceWords(
|
void UnigramDictionary::getMistypedSpaceWords(
|
||||||
const int inputLength, const int spaceProximityPos, Correction *correction,
|
const int inputLength, const int spaceProximityPos, ProximityInfo *proximityInfo,
|
||||||
const bool useFullEditDistance) {
|
Correction *correction, const bool useFullEditDistance, WordsPriorityQueue *queue) {
|
||||||
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
||||||
-1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */,
|
-1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */,
|
||||||
useFullEditDistance);
|
useFullEditDistance);
|
||||||
getSplitTwoWordsSuggestion(inputLength, correction);
|
getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c,
|
inline void UnigramDictionary::onTerminal(
|
||||||
const int inputIndex, const int skipPos, const int depth) {
|
const int freq, Correction *correction, WordsPriorityQueue *queue) {
|
||||||
const unsigned short userTypedChar = mProximityInfo->getPrimaryCharAt(inputIndex);
|
|
||||||
// Skip the ' or other letter and continue deeper
|
|
||||||
return (c == QUOTE && userTypedChar != QUOTE) || skipPos == depth;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline void UnigramDictionary::onTerminal(const int freq, Correction *correction) {
|
|
||||||
int wordLength;
|
int wordLength;
|
||||||
unsigned short* wordPointer;
|
unsigned short* wordPointer;
|
||||||
const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength);
|
const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength);
|
||||||
if (finalFreq >= 0) {
|
if (finalFreq >= 0) {
|
||||||
addWord(wordPointer, wordLength, finalFreq);
|
addWord(wordPointer, wordLength, finalFreq, queue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UnigramDictionary::getSplitTwoWordsSuggestion(
|
void UnigramDictionary::getSplitTwoWordsSuggestion(
|
||||||
const int inputLength, Correction* correction) {
|
const int inputLength, ProximityInfo *proximityInfo, Correction *correction,
|
||||||
|
WordsPriorityQueue *queue) {
|
||||||
const int spaceProximityPos = correction->getSpaceProximityPos();
|
const int spaceProximityPos = correction->getSpaceProximityPos();
|
||||||
const int missingSpacePos = correction->getMissingSpacePos();
|
const int missingSpacePos = correction->getMissingSpacePos();
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
|
@ -347,7 +345,8 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
|
||||||
const int newWordLength = firstWordLength + secondWordLength + 1;
|
const int newWordLength = firstWordLength + secondWordLength + 1;
|
||||||
// Allocating variable length array on stack
|
// Allocating variable length array on stack
|
||||||
unsigned short word[newWordLength];
|
unsigned short word[newWordLength];
|
||||||
const int firstFreq = getMostFrequentWordLike(firstWordStartPos, firstWordLength, mWord);
|
const int firstFreq = getMostFrequentWordLike(
|
||||||
|
firstWordStartPos, firstWordLength, proximityInfo, mWord);
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("First freq: %d", firstFreq);
|
LOGI("First freq: %d", firstFreq);
|
||||||
}
|
}
|
||||||
|
@ -357,7 +356,8 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
|
||||||
word[i] = mWord[i];
|
word[i] = mWord[i];
|
||||||
}
|
}
|
||||||
|
|
||||||
const int secondFreq = getMostFrequentWordLike(secondWordStartPos, secondWordLength, mWord);
|
const int secondFreq = getMostFrequentWordLike(
|
||||||
|
secondWordStartPos, secondWordLength, proximityInfo, mWord);
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("Second freq: %d", secondFreq);
|
LOGI("Second freq: %d", secondFreq);
|
||||||
}
|
}
|
||||||
|
@ -368,22 +368,22 @@ void UnigramDictionary::getSplitTwoWordsSuggestion(
|
||||||
word[i] = mWord[i - firstWordLength - 1];
|
word[i] = mWord[i - firstWordLength - 1];
|
||||||
}
|
}
|
||||||
|
|
||||||
const int pairFreq = mCorrection->getFreqForSplitTwoWords(firstFreq, secondFreq, word);
|
const int pairFreq = correction->getFreqForSplitTwoWords(firstFreq, secondFreq, word);
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength);
|
LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength);
|
||||||
}
|
}
|
||||||
addWord(word, newWordLength, pairFreq);
|
addWord(word, newWordLength, pairFreq, queue);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
|
// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
|
||||||
// interface.
|
// interface.
|
||||||
inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex,
|
inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex,
|
||||||
const int inputLength, unsigned short *word) {
|
const int inputLength, ProximityInfo *proximityInfo, unsigned short *word) {
|
||||||
uint16_t inWord[inputLength];
|
uint16_t inWord[inputLength];
|
||||||
|
|
||||||
for (int i = 0; i < inputLength; ++i) {
|
for (int i = 0; i < inputLength; ++i) {
|
||||||
inWord[i] = (uint16_t)mProximityInfo->getPrimaryCharAt(startInputIndex + i);
|
inWord[i] = (uint16_t)proximityInfo->getPrimaryCharAt(startInputIndex + i);
|
||||||
}
|
}
|
||||||
return getMostFrequentWordLikeInner(inWord, inputLength, word);
|
return getMostFrequentWordLikeInner(inWord, inputLength, word);
|
||||||
}
|
}
|
||||||
|
@ -534,7 +534,7 @@ int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offs
|
||||||
// given level, as output into newCount when traversing this level's parent.
|
// given level, as output into newCount when traversing this level's parent.
|
||||||
inline bool UnigramDictionary::processCurrentNode(const int initialPos,
|
inline bool UnigramDictionary::processCurrentNode(const int initialPos,
|
||||||
Correction *correction, int *newCount,
|
Correction *correction, int *newCount,
|
||||||
int *newChildrenPosition, int *nextSiblingPosition) {
|
int *newChildrenPosition, int *nextSiblingPosition, WordsPriorityQueue *queue) {
|
||||||
if (DEBUG_DICT) {
|
if (DEBUG_DICT) {
|
||||||
correction->checkState();
|
correction->checkState();
|
||||||
}
|
}
|
||||||
|
@ -613,7 +613,7 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos,
|
||||||
// The frequency should be here, because we come here only if this is actually
|
// The frequency should be here, because we come here only if this is actually
|
||||||
// a terminal node, and we are on its last char.
|
// a terminal node, and we are on its last char.
|
||||||
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
|
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
|
||||||
onTerminal(freq, mCorrection);
|
onTerminal(freq, correction, queue);
|
||||||
}
|
}
|
||||||
|
|
||||||
// If there are more chars in this node, then this virtual node has children.
|
// If there are more chars in this node, then this virtual node has children.
|
||||||
|
|
|
@ -66,7 +66,8 @@ public:
|
||||||
const bool isLatestDictVersion);
|
const bool isLatestDictVersion);
|
||||||
bool isValidWord(const uint16_t* const inWord, const int length) const;
|
bool isValidWord(const uint16_t* const inWord, const int length) const;
|
||||||
int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
|
int getBigramPosition(int pos, unsigned short *word, int offset, int length) const;
|
||||||
int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
int getSuggestions(ProximityInfo *proximityInfo, WordsPriorityQueue *queue,
|
||||||
|
Correction *correction, const int *xcoordinates,
|
||||||
const int *ycoordinates, const int *codes, const int codesSize, const int flags,
|
const int *ycoordinates, const int *codes, const int codesSize, const int flags,
|
||||||
unsigned short *outWords, int *frequencies);
|
unsigned short *outWords, int *frequencies);
|
||||||
virtual ~UnigramDictionary();
|
virtual ~UnigramDictionary();
|
||||||
|
@ -74,32 +75,38 @@ public:
|
||||||
private:
|
private:
|
||||||
|
|
||||||
void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
||||||
const int *ycoordinates, const int *codes, const int codesSize, const int flags);
|
const int *ycoordinates, const int *codes, const int inputLength,
|
||||||
bool isDigraph(const int* codes, const int i, const int codesSize) const;
|
const int flags, Correction *correction, WordsPriorityQueue *queue);
|
||||||
|
bool isDigraph(const int *codes, const int i, const int codesSize) const;
|
||||||
void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
|
void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
|
||||||
const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
|
const int *xcoordinates, const int* ycoordinates, const int *codesBuffer,
|
||||||
const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain,
|
const int codesBufferSize, const int flags, const int* codesSrc,
|
||||||
const int currentDepth, int* codesDest);
|
const int codesRemain, const int currentDepth, int* codesDest, Correction *correction,
|
||||||
|
WordsPriorityQueue* queue);
|
||||||
void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
|
||||||
const int *ycoordinates, const int *codes, const int codesSize);
|
const int *ycoordinates, const int *codes, const int codesSize,
|
||||||
void getSuggestionCandidates(const bool useFullEditDistance);
|
WordsPriorityQueue *queue);
|
||||||
void addWord(unsigned short *word, int length, int frequency);
|
void getSuggestionCandidates(
|
||||||
void getSplitTwoWordsSuggestion(const int inputLength, Correction *correction);
|
const bool useFullEditDistance, const int inputLength, Correction *correction,
|
||||||
|
WordsPriorityQueue* queue);
|
||||||
|
void getSplitTwoWordsSuggestion(const int inputLength, ProximityInfo *proximityInfo,
|
||||||
|
Correction *correction, WordsPriorityQueue *queue);
|
||||||
void getMissingSpaceWords(const int inputLength, const int missingSpacePos,
|
void getMissingSpaceWords(const int inputLength, const int missingSpacePos,
|
||||||
Correction *correction, const bool useFullEditDistance);
|
ProximityInfo *proximityInfo, Correction *correction,
|
||||||
|
const bool useFullEditDistance, WordsPriorityQueue *queue);
|
||||||
void getMistypedSpaceWords(const int inputLength, const int spaceProximityPos,
|
void getMistypedSpaceWords(const int inputLength, const int spaceProximityPos,
|
||||||
Correction *correction, const bool useFullEditDistance);
|
ProximityInfo *proximityInfo, Correction *correction,
|
||||||
void onTerminal(const int freq, Correction *correction);
|
const bool useFullEditDistance, WordsPriorityQueue *queue);
|
||||||
|
void onTerminal(const int freq, Correction *correction, WordsPriorityQueue *queue);
|
||||||
bool needsToSkipCurrentNode(const unsigned short c,
|
bool needsToSkipCurrentNode(const unsigned short c,
|
||||||
const int inputIndex, const int skipPos, const int depth);
|
const int inputIndex, const int skipPos, const int depth);
|
||||||
// Process a node by considering proximity, missing and excessive character
|
// Process a node by considering proximity, missing and excessive character
|
||||||
bool processCurrentNode(const int initialPos,
|
bool processCurrentNode(const int initialPos, Correction *correction, int *newCount,
|
||||||
Correction *correction, int *newCount,
|
int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueue *queue);
|
||||||
int *newChildPosition, int *nextSiblingPosition);
|
|
||||||
int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
|
int getMostFrequentWordLike(const int startInputIndex, const int inputLength,
|
||||||
unsigned short *word);
|
ProximityInfo *proximityInfo, unsigned short *word);
|
||||||
int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
|
int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length,
|
||||||
short unsigned int* outWord);
|
short unsigned int *outWord);
|
||||||
|
|
||||||
const uint8_t* const DICT_ROOT;
|
const uint8_t* const DICT_ROOT;
|
||||||
const int MAX_WORD_LENGTH;
|
const int MAX_WORD_LENGTH;
|
||||||
|
@ -122,13 +129,8 @@ private:
|
||||||
};
|
};
|
||||||
static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
|
static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[];
|
||||||
|
|
||||||
WordsPriorityQueue *mWordsPriorityQueue;
|
// Still bundled members
|
||||||
ProximityInfo *mProximityInfo;
|
unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
||||||
Correction *mCorrection;
|
|
||||||
int mInputLength;
|
|
||||||
// MAX_WORD_LENGTH_INTERNAL must be bigger than MAX_WORD_LENGTH
|
|
||||||
unsigned short mWord[MAX_WORD_LENGTH_INTERNAL];
|
|
||||||
|
|
||||||
int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
||||||
int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
||||||
int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];// TODO: remove
|
||||||
|
|
Loading…
Reference in a new issue