2009-03-13 22:11:42 +00:00
|
|
|
/*
|
|
|
|
**
|
|
|
|
** Copyright 2009, The Android Open Source Project
|
|
|
|
**
|
|
|
|
** Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
** you may not use this file except in compliance with the License.
|
|
|
|
** You may obtain a copy of the License at
|
|
|
|
**
|
|
|
|
** http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
**
|
|
|
|
** Unless required by applicable law or agreed to in writing, software
|
|
|
|
** distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
** See the License for the specific language governing permissions and
|
|
|
|
** limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <sys/mman.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <cutils/log.h>
|
|
|
|
|
|
|
|
//#define USE_ASSET_MANAGER
|
|
|
|
|
|
|
|
#ifdef USE_ASSET_MANAGER
|
|
|
|
#include <utils/AssetManager.h>
|
|
|
|
#include <utils/Asset.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include "dictionary.h"
|
|
|
|
#include "basechars.h"
|
2010-04-20 17:35:47 +00:00
|
|
|
#include "char_utils.h"
|
2009-03-13 22:11:42 +00:00
|
|
|
|
|
|
|
#define DEBUG_DICT 0
|
|
|
|
|
|
|
|
namespace latinime {
|
|
|
|
|
|
|
|
Dictionary::Dictionary(void *dict, int typedLetterMultiplier, int fullWordMultiplier)
|
|
|
|
{
|
|
|
|
mDict = (unsigned char*) dict;
|
|
|
|
mTypedLetterMultiplier = typedLetterMultiplier;
|
|
|
|
mFullWordMultiplier = fullWordMultiplier;
|
|
|
|
}
|
|
|
|
|
|
|
|
Dictionary::~Dictionary()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
int Dictionary::getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
|
2010-02-05 22:07:04 +00:00
|
|
|
int maxWordLength, int maxWords, int maxAlternatives, int skipPos,
|
|
|
|
int *nextLetters, int nextLettersSize)
|
2009-03-13 22:11:42 +00:00
|
|
|
{
|
2009-10-14 23:10:32 +00:00
|
|
|
int suggWords;
|
2009-03-13 22:11:42 +00:00
|
|
|
mFrequencies = frequencies;
|
|
|
|
mOutputChars = outWords;
|
|
|
|
mInputCodes = codes;
|
|
|
|
mInputLength = codesSize;
|
|
|
|
mMaxAlternatives = maxAlternatives;
|
|
|
|
mMaxWordLength = maxWordLength;
|
|
|
|
mMaxWords = maxWords;
|
2009-06-04 19:20:45 +00:00
|
|
|
mSkipPos = skipPos;
|
2009-07-16 01:30:47 +00:00
|
|
|
mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2;
|
2010-02-05 22:07:04 +00:00
|
|
|
mNextLettersFrequencies = nextLetters;
|
|
|
|
mNextLettersSize = nextLettersSize;
|
2009-03-13 22:11:42 +00:00
|
|
|
|
2009-07-16 01:30:47 +00:00
|
|
|
getWordsRec(0, 0, mInputLength * 3, false, 1, 0, 0);
|
2009-03-13 22:11:42 +00:00
|
|
|
|
2009-10-14 23:10:32 +00:00
|
|
|
// Get the word count
|
|
|
|
suggWords = 0;
|
|
|
|
while (suggWords < mMaxWords && mFrequencies[suggWords] > 0) suggWords++;
|
|
|
|
if (DEBUG_DICT) LOGI("Returning %d words", suggWords);
|
2010-02-05 22:07:04 +00:00
|
|
|
|
|
|
|
if (DEBUG_DICT) {
|
|
|
|
LOGI("Next letters: ");
|
|
|
|
for (int k = 0; k < nextLettersSize; k++) {
|
|
|
|
if (mNextLettersFrequencies[k] > 0) {
|
|
|
|
LOGI("%c = %d,", k, mNextLettersFrequencies[k]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
LOGI("\n");
|
|
|
|
}
|
2009-10-14 23:10:32 +00:00
|
|
|
return suggWords;
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
|
2010-02-05 22:07:04 +00:00
|
|
|
void
|
|
|
|
Dictionary::registerNextLetter(unsigned short c)
|
|
|
|
{
|
|
|
|
if (c < mNextLettersSize) {
|
|
|
|
mNextLettersFrequencies[c]++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-03-13 22:11:42 +00:00
|
|
|
unsigned short
|
|
|
|
Dictionary::getChar(int *pos)
|
|
|
|
{
|
|
|
|
unsigned short ch = (unsigned short) (mDict[(*pos)++] & 0xFF);
|
|
|
|
// If the code is 255, then actual 16 bit code follows (in big endian)
|
|
|
|
if (ch == 0xFF) {
|
|
|
|
ch = ((mDict[*pos] & 0xFF) << 8) | (mDict[*pos + 1] & 0xFF);
|
|
|
|
(*pos) += 2;
|
|
|
|
}
|
|
|
|
return ch;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
Dictionary::getAddress(int *pos)
|
|
|
|
{
|
|
|
|
int address = 0;
|
2009-03-31 17:51:17 +00:00
|
|
|
if ((mDict[*pos] & FLAG_ADDRESS_MASK) == 0) {
|
|
|
|
*pos += 1;
|
|
|
|
} else {
|
|
|
|
address += (mDict[*pos] & (ADDRESS_MASK >> 16)) << 16;
|
|
|
|
address += (mDict[*pos + 1] & 0xFF) << 8;
|
|
|
|
address += (mDict[*pos + 2] & 0xFF);
|
|
|
|
*pos += 3;
|
|
|
|
}
|
2009-03-13 22:11:42 +00:00
|
|
|
return address;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
Dictionary::wideStrLen(unsigned short *str)
|
|
|
|
{
|
|
|
|
if (!str) return 0;
|
|
|
|
unsigned short *end = str;
|
|
|
|
while (*end)
|
|
|
|
end++;
|
|
|
|
return end - str;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
Dictionary::addWord(unsigned short *word, int length, int frequency)
|
|
|
|
{
|
|
|
|
word[length] = 0;
|
2009-07-16 01:30:47 +00:00
|
|
|
if (DEBUG_DICT) {
|
|
|
|
char s[length + 1];
|
|
|
|
for (int i = 0; i <= length; i++) s[i] = word[i];
|
|
|
|
LOGI("Found word = %s, freq = %d : \n", s, frequency);
|
|
|
|
}
|
2009-03-13 22:11:42 +00:00
|
|
|
|
|
|
|
// Find the right insertion point
|
|
|
|
int insertAt = 0;
|
|
|
|
while (insertAt < mMaxWords) {
|
|
|
|
if (frequency > mFrequencies[insertAt]
|
|
|
|
|| (mFrequencies[insertAt] == frequency
|
|
|
|
&& length < wideStrLen(mOutputChars + insertAt * mMaxWordLength))) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
insertAt++;
|
|
|
|
}
|
|
|
|
if (insertAt < mMaxWords) {
|
|
|
|
memmove((char*) mFrequencies + (insertAt + 1) * sizeof(mFrequencies[0]),
|
|
|
|
(char*) mFrequencies + insertAt * sizeof(mFrequencies[0]),
|
|
|
|
(mMaxWords - insertAt - 1) * sizeof(mFrequencies[0]));
|
|
|
|
mFrequencies[insertAt] = frequency;
|
|
|
|
memmove((char*) mOutputChars + (insertAt + 1) * mMaxWordLength * sizeof(short),
|
|
|
|
(char*) mOutputChars + (insertAt ) * mMaxWordLength * sizeof(short),
|
|
|
|
(mMaxWords - insertAt - 1) * sizeof(short) * mMaxWordLength);
|
|
|
|
unsigned short *dest = mOutputChars + (insertAt ) * mMaxWordLength;
|
|
|
|
while (length--) {
|
|
|
|
*dest++ = *word++;
|
|
|
|
}
|
|
|
|
*dest = 0; // NULL terminate
|
|
|
|
if (DEBUG_DICT) LOGI("Added word at %d\n", insertAt);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned short
|
2009-08-07 21:04:24 +00:00
|
|
|
Dictionary::toLowerCase(unsigned short c) {
|
2009-03-13 22:11:42 +00:00
|
|
|
if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
|
|
|
|
c = BASE_CHARS[c];
|
|
|
|
}
|
2009-08-07 21:04:24 +00:00
|
|
|
if (c >='A' && c <= 'Z') {
|
|
|
|
c |= 32;
|
|
|
|
} else if (c > 127) {
|
2010-04-20 17:35:47 +00:00
|
|
|
c = latin_tolower(c);
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
return c;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
Dictionary::sameAsTyped(unsigned short *word, int length)
|
|
|
|
{
|
|
|
|
if (length != mInputLength) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
int *inputCodes = mInputCodes;
|
|
|
|
while (length--) {
|
|
|
|
if ((unsigned int) *inputCodes != (unsigned int) *word) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
inputCodes += mMaxAlternatives;
|
|
|
|
word++;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static char QUOTE = '\'';
|
|
|
|
|
|
|
|
void
|
2009-07-16 01:30:47 +00:00
|
|
|
Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int snr, int inputIndex,
|
|
|
|
int diffs)
|
2009-03-13 22:11:42 +00:00
|
|
|
{
|
|
|
|
// Optimization: Prune out words that are too long compared to how much was typed.
|
|
|
|
if (depth > maxDepth) {
|
|
|
|
return;
|
|
|
|
}
|
2009-07-16 01:30:47 +00:00
|
|
|
if (diffs > mMaxEditDistance) {
|
|
|
|
return;
|
|
|
|
}
|
2009-03-13 22:11:42 +00:00
|
|
|
int count = getCount(&pos);
|
|
|
|
int *currentChars = NULL;
|
|
|
|
if (mInputLength <= inputIndex) {
|
|
|
|
completion = true;
|
|
|
|
} else {
|
|
|
|
currentChars = mInputCodes + (inputIndex * mMaxAlternatives);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int i = 0; i < count; i++) {
|
|
|
|
unsigned short c = getChar(&pos);
|
2009-08-07 21:04:24 +00:00
|
|
|
unsigned short lowerC = toLowerCase(c);
|
2009-03-13 22:11:42 +00:00
|
|
|
bool terminal = getTerminal(&pos);
|
|
|
|
int childrenAddress = getAddress(&pos);
|
2009-03-31 17:51:17 +00:00
|
|
|
int freq = 1;
|
|
|
|
if (terminal) freq = getFreq(&pos);
|
2009-03-13 22:11:42 +00:00
|
|
|
// If we are only doing completions, no need to look at the typed characters.
|
|
|
|
if (completion) {
|
|
|
|
mWord[depth] = c;
|
|
|
|
if (terminal) {
|
|
|
|
addWord(mWord, depth + 1, freq * snr);
|
2010-02-05 22:07:04 +00:00
|
|
|
if (depth >= mInputLength && mSkipPos < 0) {
|
|
|
|
registerNextLetter(mWord[mInputLength]);
|
|
|
|
}
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
if (childrenAddress != 0) {
|
|
|
|
getWordsRec(childrenAddress, depth + 1, maxDepth,
|
2009-07-16 01:30:47 +00:00
|
|
|
completion, snr, inputIndex, diffs);
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
2009-06-04 19:20:45 +00:00
|
|
|
} else if (c == QUOTE && currentChars[0] != QUOTE || mSkipPos == depth) {
|
|
|
|
// Skip the ' or other letter and continue deeper
|
|
|
|
mWord[depth] = c;
|
2009-03-13 22:11:42 +00:00
|
|
|
if (childrenAddress != 0) {
|
2009-07-16 01:30:47 +00:00
|
|
|
getWordsRec(childrenAddress, depth + 1, maxDepth, false, snr, inputIndex, diffs);
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
int j = 0;
|
|
|
|
while (currentChars[j] > 0) {
|
|
|
|
if (currentChars[j] == lowerC || currentChars[j] == c) {
|
2009-07-16 01:30:47 +00:00
|
|
|
int addedWeight = j == 0 ? mTypedLetterMultiplier : 1;
|
2009-03-13 22:11:42 +00:00
|
|
|
mWord[depth] = c;
|
|
|
|
if (mInputLength == inputIndex + 1) {
|
|
|
|
if (terminal) {
|
|
|
|
if (//INCLUDE_TYPED_WORD_IF_VALID ||
|
|
|
|
!sameAsTyped(mWord, depth + 1)) {
|
2009-08-11 00:22:39 +00:00
|
|
|
int finalFreq = freq * snr * addedWeight;
|
|
|
|
if (mSkipPos < 0) finalFreq *= mFullWordMultiplier;
|
|
|
|
addWord(mWord, depth + 1, finalFreq);
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (childrenAddress != 0) {
|
|
|
|
getWordsRec(childrenAddress, depth + 1,
|
2009-07-16 01:30:47 +00:00
|
|
|
maxDepth, true, snr * addedWeight, inputIndex + 1,
|
|
|
|
diffs + (j > 0));
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
} else if (childrenAddress != 0) {
|
|
|
|
getWordsRec(childrenAddress, depth + 1, maxDepth,
|
2009-07-16 01:30:47 +00:00
|
|
|
false, snr * addedWeight, inputIndex + 1, diffs + (j > 0));
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
j++;
|
2009-06-04 19:20:45 +00:00
|
|
|
if (mSkipPos >= 0) break;
|
2009-03-13 22:11:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
Dictionary::isValidWord(unsigned short *word, int length)
|
|
|
|
{
|
|
|
|
return isValidWordRec(0, word, 0, length);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) {
|
|
|
|
int count = getCount(&pos);
|
|
|
|
unsigned short currentChar = (unsigned short) word[offset];
|
|
|
|
for (int j = 0; j < count; j++) {
|
|
|
|
unsigned short c = getChar(&pos);
|
|
|
|
int terminal = getTerminal(&pos);
|
|
|
|
int childPos = getAddress(&pos);
|
|
|
|
if (c == currentChar) {
|
|
|
|
if (offset == length - 1) {
|
|
|
|
if (terminal) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (childPos != 0) {
|
|
|
|
if (isValidWordRec(childPos, word, offset + 1, length)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2009-03-31 17:51:17 +00:00
|
|
|
if (terminal) {
|
|
|
|
getFreq(&pos);
|
|
|
|
}
|
2009-03-13 22:11:42 +00:00
|
|
|
// There could be two instances of each alphabet - upper and lower case. So continue
|
|
|
|
// looking ...
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
} // namespace latinime
|