2010-12-01 12:22:15 +00:00
|
|
|
/*
|
|
|
|
**
|
|
|
|
** Copyright 2010, The Android Open Source Project
|
|
|
|
**
|
|
|
|
** Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
** you may not use this file except in compliance with the License.
|
|
|
|
** You may obtain a copy of the License at
|
|
|
|
**
|
|
|
|
** http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
**
|
|
|
|
** Unless required by applicable law or agreed to in writing, software
|
|
|
|
** distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
** See the License for the specific language governing permissions and
|
|
|
|
** limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2010-12-06 08:38:58 +00:00
|
|
|
#include <assert.h>
|
2010-12-01 12:22:15 +00:00
|
|
|
#include <string.h>
|
|
|
|
|
2010-12-02 05:53:24 +00:00
|
|
|
#define LOG_TAG "LatinIME: unigram_dictionary.cpp"
|
2010-12-01 12:22:15 +00:00
|
|
|
|
|
|
|
#include "char_utils.h"
|
2010-12-02 05:53:24 +00:00
|
|
|
#include "dictionary.h"
|
|
|
|
#include "unigram_dictionary.h"
|
2010-12-01 12:22:15 +00:00
|
|
|
|
2011-06-28 11:45:05 +00:00
|
|
|
#include "binary_format.h"
|
|
|
|
|
2010-12-01 12:22:15 +00:00
|
|
|
namespace latinime {
|
|
|
|
|
2011-02-25 08:56:53 +00:00
|
|
|
const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] =
|
|
|
|
{ { 'a', 'e' },
|
|
|
|
{ 'o', 'e' },
|
|
|
|
{ 'u', 'e' } };
|
|
|
|
|
2011-06-16 11:55:16 +00:00
|
|
|
// TODO: check the header
|
|
|
|
UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier,
|
2010-12-08 08:05:39 +00:00
|
|
|
int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars,
|
2010-12-02 09:11:54 +00:00
|
|
|
const bool isLatestDictVersion)
|
2011-06-28 11:45:05 +00:00
|
|
|
: DICT_ROOT(streamStart + NEW_DICTIONARY_HEADER_SIZE),
|
2011-06-16 11:55:16 +00:00
|
|
|
MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords),
|
2010-12-08 08:05:39 +00:00
|
|
|
MAX_PROXIMITY_CHARS(maxProximityChars), IS_LATEST_DICT_VERSION(isLatestDictVersion),
|
|
|
|
TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier),
|
2011-06-28 11:45:05 +00:00
|
|
|
// TODO : remove this variable.
|
|
|
|
ROOT_POS(0),
|
2011-07-13 01:32:02 +00:00
|
|
|
BYTES_IN_ONE_CHAR(MAX_PROXIMITY_CHARS * sizeof(int)),
|
2011-03-04 03:17:48 +00:00
|
|
|
MAX_UMLAUT_SEARCH_DEPTH(DEFAULT_MAX_UMLAUT_SEARCH_DEPTH) {
|
2011-03-19 00:16:42 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("UnigramDictionary - constructor");
|
|
|
|
}
|
2010-12-01 12:22:15 +00:00
|
|
|
}
|
|
|
|
|
2011-07-15 04:49:00 +00:00
|
|
|
UnigramDictionary::~UnigramDictionary() {
|
|
|
|
}
|
2010-12-01 12:22:15 +00:00
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize,
|
2011-02-25 08:56:53 +00:00
|
|
|
const int MAX_PROXIMITY_CHARS) {
|
|
|
|
return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize;
|
|
|
|
}
|
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
// TODO: This needs to take an const unsigned short* and not tinker with its contents
|
|
|
|
static inline void addWord(
|
|
|
|
unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) {
|
|
|
|
queue->push(frequency, word, length);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool UnigramDictionary::isDigraph(const int *codes, const int i, const int codesSize) const {
|
2011-02-25 08:56:53 +00:00
|
|
|
|
|
|
|
// There can't be a digraph if we don't have at least 2 characters to examine
|
|
|
|
if (i + 2 > codesSize) return false;
|
|
|
|
|
|
|
|
// Search for the first char of some digraph
|
|
|
|
int lastDigraphIndex = -1;
|
|
|
|
const int thisChar = codes[i * MAX_PROXIMITY_CHARS];
|
|
|
|
for (lastDigraphIndex = sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]) - 1;
|
|
|
|
lastDigraphIndex >= 0; --lastDigraphIndex) {
|
|
|
|
if (thisChar == GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].first) break;
|
|
|
|
}
|
|
|
|
// No match: return early
|
|
|
|
if (lastDigraphIndex < 0) return false;
|
|
|
|
|
|
|
|
// It's an interesting digraph if the second char matches too.
|
|
|
|
return GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].second == codes[(i + 1) * MAX_PROXIMITY_CHARS];
|
|
|
|
}
|
|
|
|
|
|
|
|
// Mostly the same arguments as the non-recursive version, except:
|
|
|
|
// codes is the original value. It points to the start of the work buffer, and gets passed as is.
|
|
|
|
// codesSize is the size of the user input (thus, it is the size of codesSrc).
|
|
|
|
// codesDest is the current point in the work buffer.
|
|
|
|
// codesSrc is the current point in the user-input, original, content-unmodified buffer.
|
|
|
|
// codesRemain is the remaining size in codesSrc.
|
2011-07-13 01:32:02 +00:00
|
|
|
void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo,
|
2011-12-14 06:04:58 +00:00
|
|
|
const int *xcoordinates, const int *ycoordinates, const int *codesBuffer,
|
|
|
|
const int codesBufferSize, const int flags, const int *codesSrc,
|
|
|
|
const int codesRemain, const int currentDepth, int *codesDest, Correction *correction,
|
2011-12-15 07:49:12 +00:00
|
|
|
WordsPriorityQueuePool *queuePool) {
|
2011-03-04 03:17:48 +00:00
|
|
|
|
|
|
|
if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) {
|
|
|
|
for (int i = 0; i < codesRemain; ++i) {
|
|
|
|
if (isDigraph(codesSrc, i, codesRemain)) {
|
|
|
|
// Found a digraph. We will try both spellings. eg. the word is "pruefen"
|
|
|
|
|
|
|
|
// Copy the word up to the first char of the digraph, then continue processing
|
|
|
|
// on the remaining part of the word, skipping the second char of the digraph.
|
|
|
|
// In our example, copy "pru" and continue running on "fen"
|
|
|
|
// Make i the index of the second char of the digraph for simplicity. Forgetting
|
|
|
|
// to do that results in an infinite recursion so take care!
|
|
|
|
++i;
|
|
|
|
memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR);
|
|
|
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
|
|
|
codesBuffer, codesBufferSize, flags,
|
|
|
|
codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1,
|
2011-12-15 07:49:12 +00:00
|
|
|
currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, correction,
|
|
|
|
queuePool);
|
2011-03-04 03:17:48 +00:00
|
|
|
|
|
|
|
// Copy the second char of the digraph in place, then continue processing on
|
|
|
|
// the remaining part of the word.
|
|
|
|
// In our example, after "pru" in the buffer copy the "e", and continue on "fen"
|
|
|
|
memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS,
|
|
|
|
BYTES_IN_ONE_CHAR);
|
|
|
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates,
|
2011-12-14 06:04:58 +00:00
|
|
|
codesBuffer, codesBufferSize, flags,
|
|
|
|
codesSrc + i * MAX_PROXIMITY_CHARS, codesRemain - i, currentDepth + 1,
|
2011-12-15 07:49:12 +00:00
|
|
|
codesDest + i * MAX_PROXIMITY_CHARS, correction, queuePool);
|
2011-03-04 03:17:48 +00:00
|
|
|
return;
|
|
|
|
}
|
2011-02-25 08:56:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// If we come here, we hit the end of the word: let's check it against the dictionary.
|
|
|
|
// In our example, we'll come here once for "prufen" and then once for "pruefen".
|
|
|
|
// If the word contains several digraphs, we'll come it for the product of them.
|
|
|
|
// eg. if the word is "ueberpruefen" we'll test, in order, against
|
|
|
|
// "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen".
|
|
|
|
const unsigned int remainingBytes = BYTES_IN_ONE_CHAR * codesRemain;
|
|
|
|
if (0 != remainingBytes)
|
|
|
|
memcpy(codesDest, codesSrc, remainingBytes);
|
|
|
|
|
|
|
|
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
2011-12-14 06:04:58 +00:00
|
|
|
(codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, flags, correction,
|
2011-12-15 07:49:12 +00:00
|
|
|
queuePool);
|
2011-02-25 08:56:53 +00:00
|
|
|
}
|
|
|
|
|
2011-12-15 07:49:12 +00:00
|
|
|
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo,
|
|
|
|
WordsPriorityQueuePool *queuePool, Correction *correction, const int *xcoordinates,
|
|
|
|
const int *ycoordinates, const int *codes, const int codesSize, const int flags,
|
|
|
|
unsigned short *outWords, int *frequencies) {
|
2011-02-25 08:56:53 +00:00
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
Correction* masterCorrection = correction;
|
2011-02-25 08:56:53 +00:00
|
|
|
if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags)
|
|
|
|
{ // Incrementally tune the word and try all possibilities
|
|
|
|
int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)];
|
|
|
|
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
|
2011-12-15 07:49:12 +00:00
|
|
|
codesSize, flags, codes, codesSize, 0, codesBuffer, masterCorrection, queuePool);
|
2011-02-25 08:56:53 +00:00
|
|
|
} else { // Normal processing
|
2011-12-14 06:04:58 +00:00
|
|
|
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, flags,
|
2011-12-15 07:49:12 +00:00
|
|
|
masterCorrection, queuePool);
|
2011-02-25 08:56:53 +00:00
|
|
|
}
|
|
|
|
|
2011-03-04 14:06:45 +00:00
|
|
|
PROF_START(20);
|
2011-12-15 07:49:12 +00:00
|
|
|
const int suggestedWordsCount =
|
|
|
|
queuePool->getMasterQueue()->outputSuggestions(frequencies, outWords);
|
2011-02-25 08:56:53 +00:00
|
|
|
|
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("Returning %d words", suggestedWordsCount);
|
2011-06-30 08:02:23 +00:00
|
|
|
/// Print the returned words
|
|
|
|
for (int j = 0; j < suggestedWordsCount; ++j) {
|
2011-07-08 05:53:50 +00:00
|
|
|
#ifdef FLAG_DBG
|
2011-12-12 11:53:22 +00:00
|
|
|
short unsigned int* w = outWords + j * MAX_WORD_LENGTH;
|
2011-06-30 08:02:23 +00:00
|
|
|
char s[MAX_WORD_LENGTH];
|
|
|
|
for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i];
|
2011-12-12 11:53:22 +00:00
|
|
|
LOGI("%s %i", s, frequencies[j]);
|
2011-07-13 23:32:57 +00:00
|
|
|
#endif
|
2011-06-30 08:02:23 +00:00
|
|
|
}
|
2011-02-25 08:56:53 +00:00
|
|
|
}
|
2011-03-04 14:06:45 +00:00
|
|
|
PROF_END(20);
|
2011-02-25 08:56:53 +00:00
|
|
|
PROF_CLOSE;
|
|
|
|
return suggestedWordsCount;
|
|
|
|
}
|
|
|
|
|
2011-07-13 01:32:02 +00:00
|
|
|
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
|
2011-12-14 06:04:58 +00:00
|
|
|
const int *xcoordinates, const int *ycoordinates, const int *codes,
|
2011-12-15 07:49:12 +00:00
|
|
|
const int inputLength, const int flags, Correction *correction,
|
|
|
|
WordsPriorityQueuePool *queuePool) {
|
|
|
|
WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
|
2011-02-25 08:56:53 +00:00
|
|
|
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_OPEN;
|
|
|
|
PROF_START(0);
|
2011-12-15 07:49:12 +00:00
|
|
|
initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, masterQueue);
|
2011-12-14 06:04:58 +00:00
|
|
|
if (DEBUG_DICT) assert(codesSize == inputLength);
|
2010-12-13 05:42:35 +00:00
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
|
|
|
|
correction->initCorrection(proximityInfo, inputLength, maxDepth);
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(0);
|
|
|
|
|
2011-09-29 09:36:56 +00:00
|
|
|
const bool useFullEditDistance = USE_FULL_EDIT_DISTANCE & flags;
|
2011-08-11 16:05:27 +00:00
|
|
|
// TODO: remove
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_START(1);
|
2011-12-15 07:49:12 +00:00
|
|
|
getSuggestionCandidates(useFullEditDistance, inputLength, correction, masterQueue);
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(1);
|
2010-12-01 12:22:15 +00:00
|
|
|
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_START(2);
|
2011-08-19 13:05:59 +00:00
|
|
|
// Note: This line is intentionally left blank
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(2);
|
2010-12-08 07:04:16 +00:00
|
|
|
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_START(3);
|
2011-08-19 13:05:59 +00:00
|
|
|
// Note: This line is intentionally left blank
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(3);
|
2010-12-01 12:22:15 +00:00
|
|
|
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_START(4);
|
2011-08-19 13:05:59 +00:00
|
|
|
// Note: This line is intentionally left blank
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(4);
|
2010-12-09 13:08:33 +00:00
|
|
|
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_START(5);
|
2010-12-08 08:05:39 +00:00
|
|
|
// Suggestions with missing space
|
2010-12-13 05:42:35 +00:00
|
|
|
if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER
|
2011-12-14 06:04:58 +00:00
|
|
|
&& inputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) {
|
|
|
|
for (int i = 1; i < inputLength; ++i) {
|
2011-03-19 00:16:42 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("--- Suggest missing space characters %d", i);
|
|
|
|
}
|
2011-12-14 06:04:58 +00:00
|
|
|
getMissingSpaceWords(
|
2011-12-15 07:49:12 +00:00
|
|
|
inputLength, i, proximityInfo, correction, useFullEditDistance, queuePool);
|
2010-12-08 08:05:39 +00:00
|
|
|
}
|
|
|
|
}
|
2011-01-05 05:13:07 +00:00
|
|
|
PROF_END(5);
|
2011-03-04 14:06:45 +00:00
|
|
|
|
|
|
|
PROF_START(6);
|
2011-06-01 08:12:25 +00:00
|
|
|
if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) {
|
2011-03-04 14:06:45 +00:00
|
|
|
// The first and last "mistyped spaces" are taken care of by excessive character handling
|
2011-12-14 06:04:58 +00:00
|
|
|
for (int i = 1; i < inputLength - 1; ++i) {
|
2011-03-19 00:16:42 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("--- Suggest words with proximity space %d", i);
|
|
|
|
}
|
2011-03-04 14:06:45 +00:00
|
|
|
const int x = xcoordinates[i];
|
|
|
|
const int y = ycoordinates[i];
|
2011-03-19 00:16:42 +00:00
|
|
|
if (DEBUG_PROXIMITY_INFO) {
|
2011-03-04 14:06:45 +00:00
|
|
|
LOGI("Input[%d] x = %d, y = %d, has space proximity = %d",
|
|
|
|
i, x, y, proximityInfo->hasSpaceProximity(x, y));
|
2011-03-19 00:16:42 +00:00
|
|
|
}
|
2011-03-04 14:06:45 +00:00
|
|
|
if (proximityInfo->hasSpaceProximity(x, y)) {
|
2011-12-15 07:49:12 +00:00
|
|
|
getMistypedSpaceWords(inputLength, i, proximityInfo, correction,
|
|
|
|
useFullEditDistance, queuePool);
|
2011-03-04 14:06:45 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
PROF_END(6);
|
2010-12-01 12:22:15 +00:00
|
|
|
}
|
|
|
|
|
2011-09-28 03:59:43 +00:00
|
|
|
void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates,
|
2011-12-14 06:04:58 +00:00
|
|
|
const int *yCoordinates, const int *codes, const int codesSize,
|
|
|
|
WordsPriorityQueue *queue) {
|
2011-03-19 00:16:42 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("initSuggest");
|
|
|
|
}
|
2011-09-28 03:59:43 +00:00
|
|
|
proximityInfo->setInputParams(codes, codesSize, xCoordinates, yCoordinates);
|
2011-12-14 06:04:58 +00:00
|
|
|
queue->clear();
|
2010-12-01 12:22:15 +00:00
|
|
|
}
|
|
|
|
|
2010-12-02 11:19:59 +00:00
|
|
|
static const char QUOTE = '\'';
|
2010-12-08 08:05:39 +00:00
|
|
|
static const char SPACE = ' ';
|
2010-12-01 12:22:15 +00:00
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance,
|
|
|
|
const int inputLength, Correction *correction, WordsPriorityQueue *queue) {
|
2011-08-19 13:05:59 +00:00
|
|
|
// TODO: Remove setCorrectionParams
|
2011-12-14 06:04:58 +00:00
|
|
|
correction->setCorrectionParams(0, 0, 0,
|
2011-12-14 12:38:11 +00:00
|
|
|
-1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance,
|
2011-12-15 05:53:19 +00:00
|
|
|
true /* doAutoCompletion */, DEFAULT_MAX_ERRORS);
|
2010-12-08 08:05:39 +00:00
|
|
|
int rootPosition = ROOT_POS;
|
2011-06-30 08:02:23 +00:00
|
|
|
// Get the number of children of root, then increment the position
|
2011-06-16 11:55:16 +00:00
|
|
|
int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
|
2011-08-10 06:44:08 +00:00
|
|
|
int outputIndex = 0;
|
2010-12-07 04:08:39 +00:00
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0));
|
2010-12-07 04:08:39 +00:00
|
|
|
|
2010-12-08 08:05:39 +00:00
|
|
|
// Depth first search
|
2011-08-10 06:44:08 +00:00
|
|
|
while (outputIndex >= 0) {
|
2011-12-14 06:04:58 +00:00
|
|
|
if (correction->initProcessState(outputIndex)) {
|
|
|
|
int siblingPos = correction->getTreeSiblingPos(outputIndex);
|
2010-12-07 04:08:39 +00:00
|
|
|
int firstChildPos;
|
2011-08-02 17:19:44 +00:00
|
|
|
|
2011-08-03 14:27:32 +00:00
|
|
|
const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos,
|
2011-12-14 06:04:58 +00:00
|
|
|
correction, &childCount, &firstChildPos, &siblingPos, queue);
|
2010-12-08 08:05:39 +00:00
|
|
|
// Update next sibling pos
|
2011-12-14 06:04:58 +00:00
|
|
|
correction->setTreeSiblingPos(outputIndex, siblingPos);
|
2011-08-10 06:44:08 +00:00
|
|
|
|
2010-12-07 04:08:39 +00:00
|
|
|
if (needsToTraverseChildrenNodes) {
|
|
|
|
// Goes to child node
|
2011-12-14 06:04:58 +00:00
|
|
|
outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos);
|
2010-12-07 04:08:39 +00:00
|
|
|
}
|
|
|
|
} else {
|
2010-12-08 07:04:16 +00:00
|
|
|
// Goes to parent sibling node
|
2011-12-14 06:04:58 +00:00
|
|
|
outputIndex = correction->getTreeParentIndex(outputIndex);
|
2010-12-07 04:08:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-08-01 10:35:27 +00:00
|
|
|
void UnigramDictionary::getMissingSpaceWords(
|
2011-12-14 06:04:58 +00:00
|
|
|
const int inputLength, const int missingSpacePos, ProximityInfo *proximityInfo,
|
2011-12-15 07:49:12 +00:00
|
|
|
Correction *correction, const bool useFullEditDistance, WordsPriorityQueuePool *queuePool) {
|
2011-08-10 05:30:10 +00:00
|
|
|
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
2011-09-29 09:36:56 +00:00
|
|
|
-1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos,
|
2011-12-15 05:53:19 +00:00
|
|
|
useFullEditDistance, false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS);
|
2011-12-15 07:49:12 +00:00
|
|
|
getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queuePool);
|
2011-04-26 05:50:54 +00:00
|
|
|
}
|
|
|
|
|
2011-08-01 10:35:27 +00:00
|
|
|
void UnigramDictionary::getMistypedSpaceWords(
|
2011-12-14 06:04:58 +00:00
|
|
|
const int inputLength, const int spaceProximityPos, ProximityInfo *proximityInfo,
|
2011-12-15 07:49:12 +00:00
|
|
|
Correction *correction, const bool useFullEditDistance, WordsPriorityQueuePool *queuePool) {
|
2011-08-10 05:30:10 +00:00
|
|
|
correction->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
|
2011-09-29 09:36:56 +00:00
|
|
|
-1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */,
|
2011-12-15 05:53:19 +00:00
|
|
|
useFullEditDistance, false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS);
|
2011-12-15 07:49:12 +00:00
|
|
|
getSplitTwoWordsSuggestion(inputLength, proximityInfo, correction, queuePool);
|
2010-12-03 07:39:16 +00:00
|
|
|
}
|
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
inline void UnigramDictionary::onTerminal(
|
|
|
|
const int freq, Correction *correction, WordsPriorityQueue *queue) {
|
2011-08-04 09:31:57 +00:00
|
|
|
int wordLength;
|
|
|
|
unsigned short* wordPointer;
|
2011-08-10 05:30:10 +00:00
|
|
|
const int finalFreq = correction->getFinalFreq(freq, &wordPointer, &wordLength);
|
2011-08-03 14:27:32 +00:00
|
|
|
if (finalFreq >= 0) {
|
2011-12-14 06:04:58 +00:00
|
|
|
addWord(wordPointer, wordLength, finalFreq, queue);
|
2011-06-17 06:36:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-08-01 10:35:27 +00:00
|
|
|
void UnigramDictionary::getSplitTwoWordsSuggestion(
|
2011-12-14 06:04:58 +00:00
|
|
|
const int inputLength, ProximityInfo *proximityInfo, Correction *correction,
|
2011-12-15 07:49:12 +00:00
|
|
|
WordsPriorityQueuePool *queuePool) {
|
|
|
|
WordsPriorityQueue *masterQueue = queuePool->getMasterQueue();
|
|
|
|
|
2011-08-10 05:30:10 +00:00
|
|
|
const int spaceProximityPos = correction->getSpaceProximityPos();
|
|
|
|
const int missingSpacePos = correction->getMissingSpacePos();
|
2011-08-01 10:35:27 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
int inputCount = 0;
|
|
|
|
if (spaceProximityPos >= 0) ++inputCount;
|
|
|
|
if (missingSpacePos >= 0) ++inputCount;
|
|
|
|
assert(inputCount <= 1);
|
|
|
|
}
|
|
|
|
const bool isSpaceProximity = spaceProximityPos >= 0;
|
|
|
|
const int firstWordStartPos = 0;
|
|
|
|
const int secondWordStartPos = isSpaceProximity ? (spaceProximityPos + 1) : missingSpacePos;
|
|
|
|
const int firstWordLength = isSpaceProximity ? spaceProximityPos : missingSpacePos;
|
|
|
|
const int secondWordLength = isSpaceProximity
|
|
|
|
? (inputLength - spaceProximityPos - 1)
|
|
|
|
: (inputLength - missingSpacePos);
|
|
|
|
|
|
|
|
if (inputLength >= MAX_WORD_LENGTH) return;
|
2011-06-30 10:47:25 +00:00
|
|
|
if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos
|
|
|
|
|| firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength)
|
2011-08-01 10:35:27 +00:00
|
|
|
return;
|
|
|
|
|
2011-06-30 10:47:25 +00:00
|
|
|
const int newWordLength = firstWordLength + secondWordLength + 1;
|
|
|
|
// Allocating variable length array on stack
|
|
|
|
unsigned short word[newWordLength];
|
2011-12-14 06:04:58 +00:00
|
|
|
const int firstFreq = getMostFrequentWordLike(
|
|
|
|
firstWordStartPos, firstWordLength, proximityInfo, mWord);
|
2011-06-30 10:47:25 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("First freq: %d", firstFreq);
|
|
|
|
}
|
2011-08-01 10:35:27 +00:00
|
|
|
if (firstFreq <= 0) return;
|
2011-06-30 10:47:25 +00:00
|
|
|
|
|
|
|
for (int i = 0; i < firstWordLength; ++i) {
|
|
|
|
word[i] = mWord[i];
|
|
|
|
}
|
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
const int secondFreq = getMostFrequentWordLike(
|
|
|
|
secondWordStartPos, secondWordLength, proximityInfo, mWord);
|
2011-06-30 10:47:25 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("Second freq: %d", secondFreq);
|
|
|
|
}
|
2011-08-01 10:35:27 +00:00
|
|
|
if (secondFreq <= 0) return;
|
2011-06-30 10:47:25 +00:00
|
|
|
|
|
|
|
word[firstWordLength] = SPACE;
|
|
|
|
for (int i = (firstWordLength + 1); i < newWordLength; ++i) {
|
|
|
|
word[i] = mWord[i - firstWordLength - 1];
|
|
|
|
}
|
|
|
|
|
2011-12-14 06:04:58 +00:00
|
|
|
const int pairFreq = correction->getFreqForSplitTwoWords(firstFreq, secondFreq, word);
|
2011-06-30 10:47:25 +00:00
|
|
|
if (DEBUG_DICT) {
|
2011-08-01 10:35:27 +00:00
|
|
|
LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength);
|
2011-06-30 10:47:25 +00:00
|
|
|
}
|
2011-12-15 07:49:12 +00:00
|
|
|
addWord(word, newWordLength, pairFreq, masterQueue);
|
2011-08-01 10:35:27 +00:00
|
|
|
return;
|
2011-06-30 10:47:25 +00:00
|
|
|
}
|
|
|
|
|
2011-06-28 11:45:05 +00:00
|
|
|
// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
|
|
|
|
// interface.
|
|
|
|
inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex,
|
2011-12-14 06:04:58 +00:00
|
|
|
const int inputLength, ProximityInfo *proximityInfo, unsigned short *word) {
|
2011-06-28 11:45:05 +00:00
|
|
|
uint16_t inWord[inputLength];
|
|
|
|
|
|
|
|
for (int i = 0; i < inputLength; ++i) {
|
2011-12-14 06:04:58 +00:00
|
|
|
inWord[i] = (uint16_t)proximityInfo->getPrimaryCharAt(startInputIndex + i);
|
2011-06-28 11:45:05 +00:00
|
|
|
}
|
|
|
|
return getMostFrequentWordLikeInner(inWord, inputLength, word);
|
|
|
|
}
|
|
|
|
|
|
|
|
// This function will take the position of a character array within a CharGroup,
|
|
|
|
// and check it actually like-matches the word in inWord starting at startInputIndex,
|
|
|
|
// that is, it matches it with case and accents squashed.
|
|
|
|
// The function returns true if there was a full match, false otherwise.
|
|
|
|
// The function will copy on-the-fly the characters in the CharGroup to outNewWord.
|
|
|
|
// It will also place the end position of the array in outPos; in outInputIndex,
|
|
|
|
// it will place the index of the first char AFTER the match if there was a match,
|
|
|
|
// and the initial position if there was not. It makes sense because if there was
|
|
|
|
// a match we want to continue searching, but if there was not, we want to go to
|
|
|
|
// the next CharGroup.
|
|
|
|
// In and out parameters may point to the same location. This function takes care
|
|
|
|
// not to use any input parameters after it wrote into its outputs.
|
|
|
|
static inline bool testCharGroupForContinuedLikeness(const uint8_t flags,
|
|
|
|
const uint8_t* const root, const int startPos,
|
|
|
|
const uint16_t* const inWord, const int startInputIndex,
|
|
|
|
int32_t* outNewWord, int* outInputIndex, int* outPos) {
|
|
|
|
const bool hasMultipleChars = (0 != (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags));
|
|
|
|
int pos = startPos;
|
|
|
|
int32_t character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
|
2011-11-11 05:26:13 +00:00
|
|
|
int32_t baseChar = toBaseLowerCase(character);
|
|
|
|
const uint16_t wChar = toBaseLowerCase(inWord[startInputIndex]);
|
2011-06-28 11:45:05 +00:00
|
|
|
|
|
|
|
if (baseChar != wChar) {
|
|
|
|
*outPos = hasMultipleChars ? BinaryFormat::skipOtherCharacters(root, pos) : pos;
|
|
|
|
*outInputIndex = startInputIndex;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
int inputIndex = startInputIndex;
|
|
|
|
outNewWord[inputIndex] = character;
|
|
|
|
if (hasMultipleChars) {
|
|
|
|
character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
|
|
|
|
while (NOT_A_CHARACTER != character) {
|
2011-11-11 05:26:13 +00:00
|
|
|
baseChar = toBaseLowerCase(character);
|
|
|
|
if (toBaseLowerCase(inWord[++inputIndex]) != baseChar) {
|
2011-06-28 11:45:05 +00:00
|
|
|
*outPos = BinaryFormat::skipOtherCharacters(root, pos);
|
|
|
|
*outInputIndex = startInputIndex;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
outNewWord[inputIndex] = character;
|
|
|
|
character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*outInputIndex = inputIndex + 1;
|
|
|
|
*outPos = pos;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// This function is invoked when a word like the word searched for is found.
|
|
|
|
// It will compare the frequency to the max frequency, and if greater, will
|
|
|
|
// copy the word into the output buffer. In output value maxFreq, it will
|
|
|
|
// write the new maximum frequency if it changed.
|
|
|
|
static inline void onTerminalWordLike(const int freq, int32_t* newWord, const int length,
|
|
|
|
short unsigned int* outWord, int* maxFreq) {
|
|
|
|
if (freq > *maxFreq) {
|
|
|
|
for (int q = 0; q < length; ++q)
|
|
|
|
outWord[q] = newWord[q];
|
|
|
|
outWord[length] = 0;
|
|
|
|
*maxFreq = freq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Will find the highest frequency of the words like the one passed as an argument,
|
|
|
|
// that is, everything that only differs by case/accents.
|
|
|
|
int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWord,
|
|
|
|
const int length, short unsigned int* outWord) {
|
|
|
|
int32_t newWord[MAX_WORD_LENGTH_INTERNAL];
|
|
|
|
int depth = 0;
|
|
|
|
int maxFreq = -1;
|
|
|
|
const uint8_t* const root = DICT_ROOT;
|
|
|
|
|
|
|
|
mStackChildCount[0] = root[0];
|
|
|
|
mStackInputIndex[0] = 0;
|
|
|
|
mStackSiblingPos[0] = 1;
|
|
|
|
while (depth >= 0) {
|
|
|
|
const int charGroupCount = mStackChildCount[depth];
|
|
|
|
int pos = mStackSiblingPos[depth];
|
|
|
|
for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) {
|
|
|
|
int inputIndex = mStackInputIndex[depth];
|
|
|
|
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
|
|
|
|
// Test whether all chars in this group match with the word we are searching for. If so,
|
|
|
|
// we want to traverse its children (or if the length match, evaluate its frequency).
|
|
|
|
// Note that this function will output the position regardless, but will only write
|
|
|
|
// into inputIndex if there is a match.
|
|
|
|
const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord,
|
|
|
|
inputIndex, newWord, &inputIndex, &pos);
|
|
|
|
if (isAlike && (FLAG_IS_TERMINAL & flags) && (inputIndex == length)) {
|
|
|
|
const int frequency = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos);
|
|
|
|
onTerminalWordLike(frequency, newWord, inputIndex, outWord, &maxFreq);
|
|
|
|
}
|
|
|
|
pos = BinaryFormat::skipFrequency(flags, pos);
|
|
|
|
const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos);
|
|
|
|
const int childrenNodePos = BinaryFormat::readChildrenPosition(root, flags, pos);
|
|
|
|
// If we had a match and the word has children, we want to traverse them. We don't have
|
|
|
|
// to traverse words longer than the one we are searching for, since they will not match
|
|
|
|
// anyway, so don't traverse unless inputIndex < length.
|
|
|
|
if (isAlike && (-1 != childrenNodePos) && (inputIndex < length)) {
|
|
|
|
// Save position for this depth, to get back to this once children are done
|
|
|
|
mStackChildCount[depth] = charGroupIndex;
|
|
|
|
mStackSiblingPos[depth] = siblingPos;
|
|
|
|
// Prepare stack values for next depth
|
|
|
|
++depth;
|
|
|
|
int childrenPos = childrenNodePos;
|
|
|
|
mStackChildCount[depth] =
|
|
|
|
BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos);
|
|
|
|
mStackSiblingPos[depth] = childrenPos;
|
|
|
|
mStackInputIndex[depth] = inputIndex;
|
|
|
|
pos = childrenPos;
|
|
|
|
// Go to the next depth level.
|
|
|
|
++depth;
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
// No match, or no children, or word too long to ever match: go the next sibling.
|
|
|
|
pos = siblingPos;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
--depth;
|
|
|
|
}
|
|
|
|
return maxFreq;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool UnigramDictionary::isValidWord(const uint16_t* const inWord, const int length) const {
|
2011-07-25 09:17:11 +00:00
|
|
|
return NOT_VALID_WORD != BinaryFormat::getTerminalPosition(DICT_ROOT, inWord, length);
|
2011-06-28 11:45:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: remove this function.
|
|
|
|
int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offset,
|
|
|
|
int length) const {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
// ProcessCurrentNode returns a boolean telling whether to traverse children nodes or not.
|
|
|
|
// If the return value is false, then the caller should read in the output "nextSiblingPosition"
|
|
|
|
// to find out the address of the next sibling node and pass it to a new call of processCurrentNode.
|
|
|
|
// It is worthy to note that when false is returned, the output values other than
|
|
|
|
// nextSiblingPosition are undefined.
|
|
|
|
// If the return value is true, then the caller must proceed to traverse the children of this
|
|
|
|
// node. processCurrentNode will output the information about the children: their count in
|
|
|
|
// newCount, their position in newChildrenPosition, the traverseAllNodes flag in
|
|
|
|
// newTraverseAllNodes, the match weight into newMatchRate, the input index into newInputIndex, the
|
|
|
|
// diffs into newDiffs, the sibling position in nextSiblingPosition, and the output index into
|
|
|
|
// newOutputIndex. Please also note the following caveat: processCurrentNode does not know when
|
|
|
|
// there aren't any more nodes at this level, it merely returns the address of the first byte after
|
|
|
|
// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any
|
|
|
|
// given level, as output into newCount when traversing this level's parent.
|
2011-08-04 09:31:57 +00:00
|
|
|
inline bool UnigramDictionary::processCurrentNode(const int initialPos,
|
2011-08-10 05:30:10 +00:00
|
|
|
Correction *correction, int *newCount,
|
2011-12-14 06:04:58 +00:00
|
|
|
int *newChildrenPosition, int *nextSiblingPosition, WordsPriorityQueue *queue) {
|
2011-06-21 13:23:21 +00:00
|
|
|
if (DEBUG_DICT) {
|
2011-08-10 05:30:10 +00:00
|
|
|
correction->checkState();
|
2011-06-21 13:23:21 +00:00
|
|
|
}
|
2011-06-30 10:23:16 +00:00
|
|
|
int pos = initialPos;
|
|
|
|
|
2011-06-28 11:45:05 +00:00
|
|
|
// Flags contain the following information:
|
|
|
|
// - Address type (MASK_GROUP_ADDRESS_TYPE) on two bits:
|
|
|
|
// - FLAG_GROUP_ADDRESS_TYPE_{ONE,TWO,THREE}_BYTES means there are children and their address
|
|
|
|
// is on the specified number of bytes.
|
|
|
|
// - FLAG_GROUP_ADDRESS_TYPE_NOADDRESS means there are no children, and therefore no address.
|
|
|
|
// - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not.
|
|
|
|
// - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children)
|
|
|
|
// - FLAG_HAS_BIGRAMS: whether this node has bigrams or not
|
|
|
|
const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos);
|
|
|
|
const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags));
|
2011-08-04 09:31:57 +00:00
|
|
|
const bool isTerminalNode = (0 != (FLAG_IS_TERMINAL & flags));
|
|
|
|
|
|
|
|
bool needsToInvokeOnTerminal = false;
|
2011-06-28 11:45:05 +00:00
|
|
|
|
|
|
|
// This gets only ONE character from the stream. Next there will be:
|
|
|
|
// if FLAG_HAS_MULTIPLE CHARS: the other characters of the same node
|
|
|
|
// else if FLAG_IS_TERMINAL: the frequency
|
|
|
|
// else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address
|
|
|
|
// Note that you can't have a node that both is not a terminal and has no children.
|
|
|
|
int32_t c = BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos);
|
|
|
|
assert(NOT_A_CHARACTER != c);
|
|
|
|
|
|
|
|
// We are going to loop through each character and make it look like it's a different
|
|
|
|
// node each time. To do that, we will process characters in this node in order until
|
|
|
|
// we find the character terminator. This is signalled by getCharCode* returning
|
|
|
|
// NOT_A_CHARACTER.
|
|
|
|
// As a special case, if there is only one character in this node, we must not read the
|
|
|
|
// next bytes so we will simulate the NOT_A_CHARACTER return by testing the flags.
|
|
|
|
// This way, each loop run will look like a "virtual node".
|
|
|
|
do {
|
|
|
|
// We prefetch the next char. If 'c' is the last char of this node, we will have
|
|
|
|
// NOT_A_CHARACTER in the next char. From this we can decide whether this virtual node
|
|
|
|
// should behave as a terminal or not and whether we have children.
|
|
|
|
const int32_t nextc = hasMultipleChars
|
|
|
|
? BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CHARACTER;
|
|
|
|
const bool isLastChar = (NOT_A_CHARACTER == nextc);
|
|
|
|
// If there are more chars in this nodes, then this virtual node is not a terminal.
|
|
|
|
// If we are on the last char, this virtual node is a terminal if this node is.
|
2011-08-04 09:31:57 +00:00
|
|
|
const bool isTerminal = isLastChar && isTerminalNode;
|
|
|
|
|
2011-08-10 05:30:10 +00:00
|
|
|
Correction::CorrectionType stateType = correction->processCharAndCalcState(
|
2011-08-04 09:31:57 +00:00
|
|
|
c, isTerminal);
|
2011-08-10 05:30:10 +00:00
|
|
|
if (stateType == Correction::TRAVERSE_ALL_ON_TERMINAL
|
|
|
|
|| stateType == Correction::ON_TERMINAL) {
|
2011-08-04 09:31:57 +00:00
|
|
|
needsToInvokeOnTerminal = true;
|
2011-12-14 12:38:11 +00:00
|
|
|
} else if (stateType == Correction::UNRELATED || correction->needsToPrune()) {
|
2011-08-04 09:31:57 +00:00
|
|
|
// We found that this is an unrelated character, so we should give up traversing
|
|
|
|
// this node and its children entirely.
|
|
|
|
// However we may not be on the last virtual node yet so we skip the remaining
|
|
|
|
// characters in this node, the frequency if it's there, read the next sibling
|
|
|
|
// position to output it, then return false.
|
|
|
|
// We don't have to output other values because we return false, as in
|
|
|
|
// "don't traverse children".
|
2011-06-28 11:45:05 +00:00
|
|
|
if (!isLastChar) {
|
|
|
|
pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos);
|
|
|
|
}
|
|
|
|
pos = BinaryFormat::skipFrequency(flags, pos);
|
|
|
|
*nextSiblingPosition =
|
|
|
|
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
|
|
|
|
return false;
|
2011-06-21 13:23:21 +00:00
|
|
|
}
|
|
|
|
|
2011-06-28 11:45:05 +00:00
|
|
|
// Prepare for the next character. Promote the prefetched char to current char - the loop
|
|
|
|
// will take care of prefetching the next. If we finally found our last char, nextc will
|
|
|
|
// contain NOT_A_CHARACTER.
|
|
|
|
c = nextc;
|
|
|
|
} while (NOT_A_CHARACTER != c);
|
2011-06-21 13:23:21 +00:00
|
|
|
|
2011-08-04 09:31:57 +00:00
|
|
|
if (isTerminalNode) {
|
|
|
|
if (needsToInvokeOnTerminal) {
|
|
|
|
// The frequency should be here, because we come here only if this is actually
|
|
|
|
// a terminal node, and we are on its last char.
|
|
|
|
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
|
2011-12-14 06:04:58 +00:00
|
|
|
onTerminal(freq, correction, queue);
|
2011-08-04 09:31:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If there are more chars in this node, then this virtual node has children.
|
|
|
|
// If we are on the last char, this virtual node has children if this node has.
|
|
|
|
const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags);
|
|
|
|
|
|
|
|
// This character matched the typed character (enough to traverse the node at least)
|
|
|
|
// so we just evaluated it. Now we should evaluate this virtual node's children - that
|
|
|
|
// is, if it has any. If it has no children, we're done here - so we skip the end of
|
|
|
|
// the node, output the siblings position, and return false "don't traverse children".
|
|
|
|
// Note that !hasChildren implies isLastChar, so we know we don't have to skip any
|
|
|
|
// remaining char in this group for there can't be any.
|
|
|
|
if (!hasChildren) {
|
|
|
|
pos = BinaryFormat::skipFrequency(flags, pos);
|
|
|
|
*nextSiblingPosition =
|
|
|
|
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
|
|
|
|
return false;
|
|
|
|
}
|
2011-06-28 11:45:05 +00:00
|
|
|
|
2011-08-04 09:31:57 +00:00
|
|
|
// Optimization: Prune out words that are too long compared to how much was typed.
|
2011-08-10 05:30:10 +00:00
|
|
|
if (correction->needsToPrune()) {
|
2011-08-04 09:31:57 +00:00
|
|
|
pos = BinaryFormat::skipFrequency(flags, pos);
|
|
|
|
*nextSiblingPosition =
|
|
|
|
BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
|
2011-08-19 13:05:59 +00:00
|
|
|
if (DEBUG_DICT_FULL) {
|
|
|
|
LOGI("Traversing was pruned.");
|
|
|
|
}
|
2011-08-04 09:31:57 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
2011-06-28 11:45:05 +00:00
|
|
|
|
|
|
|
// Now we finished processing this node, and we want to traverse children. If there are no
|
|
|
|
// children, we can't come here.
|
|
|
|
assert(BinaryFormat::hasChildrenInFlags(flags));
|
|
|
|
|
|
|
|
// If this node was a terminal it still has the frequency under the pointer (it may have been
|
|
|
|
// read, but not skipped - see readFrequencyWithoutMovingPointer).
|
|
|
|
// Next come the children position, then possibly attributes (attributes are bigrams only for
|
|
|
|
// now, maybe something related to shortcuts in the future).
|
|
|
|
// Once this is read, we still need to output the number of nodes in the immediate children of
|
|
|
|
// this node, so we read and output it before returning true, as in "please traverse children".
|
|
|
|
pos = BinaryFormat::skipFrequency(flags, pos);
|
|
|
|
int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos);
|
|
|
|
*nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos);
|
|
|
|
*newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos);
|
|
|
|
*newChildrenPosition = childrenPos;
|
|
|
|
return true;
|
2011-06-21 13:23:21 +00:00
|
|
|
}
|
|
|
|
|
2010-12-01 12:22:15 +00:00
|
|
|
} // namespace latinime
|