nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Remove MAX_WORD_LENGTH_INTERNAL"

Browse source
This commit is contained in:
Ken Wakasa 2013-01-14 19:46:09 -08:00 committed by Android (Google) Code Review
commit c995dbcf59
25 changed files with 203 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
java/src/com/android/inputmethod/keyboard/ProximityInfo.java
View file

@ -80,17 +80,17 @@ public final class ProximityInfo {
mNativeProximityInfo = createNativeProximityInfo(touchPositionCorrection);
}
public static ProximityInfo createDummyProximityInfo() {
private static ProximityInfo createDummyProximityInfo() {
return new ProximityInfo("", 1, 1, 1, 1, 1, 1, EMPTY_KEY_ARRAY, null);
}
public static ProximityInfo createSpellCheckerProximityInfo(final int[] proximity,
public static ProximityInfo createSpellCheckerProximityInfo(final int[] proximityCharsArray,
final int rowSize, final int gridWidth, final int gridHeight) {
final ProximityInfo spellCheckerProximityInfo = createDummyProximityInfo();
spellCheckerProximityInfo.mNativeProximityInfo =
spellCheckerProximityInfo.setProximityInfoNative("",
rowSize, gridWidth, gridHeight, gridWidth, gridHeight,
1, proximity, 0, null, null, null, null, null, null, null, null);
1, proximityCharsArray, 0, null, null, null, null, null, null, null, null);
return spellCheckerProximityInfo;
}
@ -100,15 +100,13 @@ public final class ProximityInfo {
}
// TODO: Stop passing proximityCharsArray
private native long setProximityInfoNative(
String locale, int maxProximityCharsSize, int displayWidth,
int displayHeight, int gridWidth, int gridHeight,
int mostCommonKeyWidth, int[] proximityCharsArray,
int keyCount, int[] keyXCoordinates, int[] keyYCoordinates,
int[] keyWidths, int[] keyHeights, int[] keyCharCodes,
private static native long setProximityInfoNative(String locale, int maxProximityCharsSize,
int displayWidth, int displayHeight, int gridWidth, int gridHeight,
int mostCommonKeyWidth, int[] proximityCharsArray, int keyCount, int[] keyXCoordinates,
int[] keyYCoordinates, int[] keyWidths, int[] keyHeights, int[] keyCharCodes,
float[] sweetSpotCenterX, float[] sweetSpotCenterY, float[] sweetSpotRadii);
private native void releaseProximityInfoNative(long nativeProximityInfo);
private static native void releaseProximityInfoNative(long nativeProximityInfo);
private static boolean needsProximityInfo(final Key key) {
// Don't include special keys into ProximityInfo.
@ -232,10 +230,10 @@ public final class ProximityInfo {
}
// TODO: Stop passing proximityCharsArray
return setProximityInfoNative(mLocaleStr, MAX_PROXIMITY_CHARS_SIZE,
mKeyboardMinWidth, mKeyboardHeight, mGridWidth, mGridHeight, mMostCommonKeyWidth,
proximityCharsArray, keyCount, keyXCoordinates, keyYCoordinates, keyWidths,
keyHeights, keyCharCodes, sweetSpotCenterXs, sweetSpotCenterYs, sweetSpotRadii);
return setProximityInfoNative(mLocaleStr, MAX_PROXIMITY_CHARS_SIZE, mKeyboardMinWidth,
mKeyboardHeight, mGridWidth, mGridHeight, mMostCommonKeyWidth, proximityCharsArray,
keyCount, keyXCoordinates, keyYCoordinates, keyWidths, keyHeights, keyCharCodes,
sweetSpotCenterXs, sweetSpotCenterYs, sweetSpotRadii);
}
public long getNativeProximityInfo() {

35
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -35,14 +35,10 @@ public final class BinaryDictionary extends Dictionary {
public static final String DICTIONARY_PACK_AUTHORITY =
"com.android.inputmethod.latin.dictionarypack";
/**
* There is a difference between what java and native code can handle.
* This value should only be used in BinaryDictionary.java
* It is necessary to keep it at this value because some languages e.g. German have
* really long words.
*/
// Must be identical to MAX_WORD_LENGTH in native/jni/src/defines.h
private static final int MAX_WORD_LENGTH = Constants.Dictionary.MAX_WORD_LENGTH;
private static final int MAX_RESULTS = 18; /* Must be identical to MAX_RESULTS in defines.h */
// Must be identical to MAX_RESULTS in native/jni/src/defines.h
private static final int MAX_RESULTS = 18;
private long mNativeDict;
private final Locale mLocale;
@ -94,23 +90,22 @@ public final class BinaryDictionary extends Dictionary {
JniUtils.loadNativeLibrary();
}
private native long openNative(String sourceDir, long dictOffset, long dictSize,
int maxWordLength);
private native void closeNative(long dict);
private native int getFrequencyNative(long dict, int[] word);
private native boolean isValidBigramNative(long dict, int[] word1, int[] word2);
private native int getSuggestionsNative(long dict, long proximityInfo, long traverseSession,
int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds,
int[] inputCodePoints, int codesSize, int commitPoint, boolean isGesture,
int[] prevWordCodePointArray, boolean useFullEditDistance, int[] outputCodePoints,
int[] outputScores, int[] outputIndices, int[] outputTypes);
private static native long openNative(String sourceDir, long dictOffset, long dictSize);
private static native void closeNative(long dict);
private static native int getFrequencyNative(long dict, int[] word);
private static native boolean isValidBigramNative(long dict, int[] word1, int[] word2);
private static native int getSuggestionsNative(long dict, long proximityInfo,
long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times,
int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint,
boolean isGesture, int[] prevWordCodePointArray, boolean useFullEditDistance,
int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes);
private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
private static native int editDistanceNative(int[] before, int[] after);
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
final long length) {
mNativeDict = openNative(path, startOffset, length, MAX_WORD_LENGTH);
mNativeDict = openNative(path, startOffset, length);
}
@Override
@ -139,12 +134,12 @@ public final class BinaryDictionary extends Dictionary {
}
final InputPointers ips = composer.getInputPointers();
final int codesSize = isGesture ? ips.getPointerSize() : composerSize;
final int inputSize = isGesture ? ips.getPointerSize() : composerSize;
// proximityInfo and/or prevWordForBigrams may not be null.
final int count = getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(),
getTraverseSession(sessionId).getSession(), ips.getXCoordinates(),
ips.getYCoordinates(), ips.getTimes(), ips.getPointerIds(), mInputCodePoints,
codesSize, 0 /* commitPoint */, isGesture, prevWordCodePointArray,
inputSize, 0 /* commitPoint */, isGesture, prevWordCodePointArray,
mUseFullEditDistance, mOutputCodePoints, mOutputScores, mSpaceIndices,
mOutputTypes);
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();

1
java/src/com/android/inputmethod/latin/Constants.java
View file

@ -127,6 +127,7 @@ public final class Constants {
}
public static final class Dictionary {
// Must be identical to MAX_WORD_LENGTH in native/jni/src/defines.h
public static final int MAX_WORD_LENGTH = 48;
private Dictionary() {

6
java/src/com/android/inputmethod/latin/DicTraverseSession.java
View file

@ -23,10 +23,10 @@ public final class DicTraverseSession {
JniUtils.loadNativeLibrary();
}
private native long setDicTraverseSessionNative(String locale);
private native void initDicTraverseSessionNative(long nativeDicTraverseSession,
private static native long setDicTraverseSessionNative(String locale);
private static native void initDicTraverseSessionNative(long nativeDicTraverseSession,
long dictionary, int[] previousWord, int previousWordLength);
private native void releaseDicTraverseSessionNative(long nativeDicTraverseSession);
private static native void releaseDicTraverseSessionNative(long nativeDicTraverseSession);
private long mNativeDicTraverseSession;

14
native/jni/com_android_inputmethod_keyboard_ProximityInfo.cpp
View file

@ -24,12 +24,12 @@
namespace latinime {
static jlong latinime_Keyboard_setProximityInfo(JNIEnv *env, jobject object,
jstring localeJStr, jint maxProximityCharsSize, jint displayWidth, jint displayHeight,
jint gridWidth, jint gridHeight, jint mostCommonkeyWidth, jintArray proximityChars,
jint keyCount, jintArray keyXCoordinates, jintArray keyYCoordinates,
jintArray keyWidths, jintArray keyHeights, jintArray keyCharCodes,
jfloatArray sweetSpotCenterXs, jfloatArray sweetSpotCenterYs, jfloatArray sweetSpotRadii) {
static jlong latinime_Keyboard_setProximityInfo(JNIEnv *env, jclass clazz, jstring localeJStr,
jint maxProximityCharsSize, jint displayWidth, jint displayHeight, jint gridWidth,
jint gridHeight, jint mostCommonkeyWidth, jintArray proximityChars, jint keyCount,
jintArray keyXCoordinates, jintArray keyYCoordinates, jintArray keyWidths,
jintArray keyHeights, jintArray keyCharCodes, jfloatArray sweetSpotCenterXs,
jfloatArray sweetSpotCenterYs, jfloatArray sweetSpotRadii) {
ProximityInfo *proximityInfo = new ProximityInfo(env, localeJStr, maxProximityCharsSize,
displayWidth, displayHeight, gridWidth, gridHeight, mostCommonkeyWidth, proximityChars,
keyCount, keyXCoordinates, keyYCoordinates, keyWidths, keyHeights, keyCharCodes,
@ -37,7 +37,7 @@ static jlong latinime_Keyboard_setProximityInfo(JNIEnv *env, jobject object,
return reinterpret_cast<jlong>(proximityInfo);
}
static void latinime_Keyboard_release(JNIEnv *env, jobject object, jlong proximityInfo) {
static void latinime_Keyboard_release(JNIEnv *env, jclass clazz, jlong proximityInfo) {
ProximityInfo *pi = reinterpret_cast<ProximityInfo *>(proximityInfo);
delete pi;
}

61
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -42,8 +42,8 @@ class ProximityInfo;
static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd);
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jobject object, jstring sourceDir,
jlong dictOffset, jlong dictSize, jint maxWordLength) {
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
jlong dictOffset, jlong dictSize) {
PROF_OPEN;
PROF_START(66);
const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
@ -117,17 +117,17 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jobject object, jstring
releaseDictBuf(dictBuf, 0, 0);
#endif // USE_MMAP_FOR_DICTIONARY
} else {
dictionary = new Dictionary(dictBuf, static_cast<int>(dictSize), fd, adjust, maxWordLength);
dictionary = new Dictionary(dictBuf, static_cast<int>(dictSize), fd, adjust);
}
PROF_END(66);
PROF_CLOSE;
return (jlong)dictionary;
}
static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object, jlong dict,
static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict,
jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray,
jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray,
jintArray inputCodePointsArray, jint arraySize, jint commitPoint, jboolean isGesture,
jintArray inputCodePointsArray, jint inputSize, jint commitPoint, jboolean isGesture,
jintArray prevWordCodePointsForBigrams, jboolean useFullEditDistance,
jintArray outputCodePointsArray, jintArray scoresArray, jintArray spaceIndicesArray,
jintArray outputTypesArray) {
@ -137,20 +137,20 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object,
void *traverseSession = reinterpret_cast<void *>(dicTraverseSession);
// Input values
int xCoordinates[arraySize];
int yCoordinates[arraySize];
int times[arraySize];
int pointerIds[arraySize];
int xCoordinates[inputSize];
int yCoordinates[inputSize];
int times[inputSize];
int pointerIds[inputSize];
const jsize inputCodePointsLength = env->GetArrayLength(inputCodePointsArray);
int inputCodePoints[inputCodePointsLength];
const jsize prevWordCodePointsLength =
prevWordCodePointsForBigrams ? env->GetArrayLength(prevWordCodePointsForBigrams) : 0;
int prevWordCodePointsInternal[prevWordCodePointsLength];
int *prevWordCodePoints = 0;
env->GetIntArrayRegion(xCoordinatesArray, 0, arraySize, xCoordinates);
env->GetIntArrayRegion(yCoordinatesArray, 0, arraySize, yCoordinates);
env->GetIntArrayRegion(timesArray, 0, arraySize, times);
env->GetIntArrayRegion(pointerIdsArray, 0, arraySize, pointerIds);
env->GetIntArrayRegion(xCoordinatesArray, 0, inputSize, xCoordinates);
env->GetIntArrayRegion(yCoordinatesArray, 0, inputSize, yCoordinates);
env->GetIntArrayRegion(timesArray, 0, inputSize, times);
env->GetIntArrayRegion(pointerIdsArray, 0, inputSize, pointerIds);
env->GetIntArrayRegion(inputCodePointsArray, 0, inputCodePointsLength, inputCodePoints);
if (prevWordCodePointsForBigrams) {
env->GetIntArrayRegion(prevWordCodePointsForBigrams, 0, prevWordCodePointsLength,
@ -159,17 +159,20 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object,
}
// Output values
const jsize outputCodePointsLength = env->GetArrayLength(outputCodePointsArray);
int outputCodePoints[outputCodePointsLength];
const jsize scoresLength = env->GetArrayLength(scoresArray);
/* By the way, let's check the output array length here to make sure */
if (scoresLength < MAX_RESULTS) {
const jsize outputCodePointsLength = env->GetArrayLength(outputCodePointsArray);
if (outputCodePointsLength != (MAX_WORD_LENGTH * MAX_RESULTS)) {
AKLOGE("Invalid outputCodePointsLength: %d", outputCodePointsLength);
ASSERT(false);
return 0;
}
// Cont'd: Output values
const jsize scoresLength = env->GetArrayLength(scoresArray);
if (scoresLength != MAX_RESULTS) {
AKLOGE("Invalid scoresLength: %d", scoresLength);
ASSERT(false);
return 0;
}
int outputCodePoints[outputCodePointsLength];
int scores[scoresLength];
const jsize spaceIndicesLength = env->GetArrayLength(spaceIndicesArray);
int spaceIndices[spaceIndicesLength];
@ -181,14 +184,14 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object,
memset(outputTypes, 0, sizeof(outputTypes));
int count;
if (isGesture || arraySize > 0) {
if (isGesture || inputSize > 0) {
count = dictionary->getSuggestions(pInfo, traverseSession, xCoordinates, yCoordinates,
times, pointerIds, inputCodePoints, arraySize, prevWordCodePoints,
times, pointerIds, inputCodePoints, inputSize, prevWordCodePoints,
prevWordCodePointsLength, commitPoint, isGesture, useFullEditDistance,
outputCodePoints, scores, spaceIndices, outputTypes);
} else {
count = dictionary->getBigrams(prevWordCodePoints, prevWordCodePointsLength,
inputCodePoints, arraySize, outputCodePoints, scores, outputTypes);
inputCodePoints, inputSize, outputCodePoints, scores, outputTypes);
}
// Copy back the output values
@ -200,7 +203,7 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object,
return count;
}
static jint latinime_BinaryDictionary_getFrequency(JNIEnv *env, jobject object, jlong dict,
static jint latinime_BinaryDictionary_getFrequency(JNIEnv *env, jclass clazz, jlong dict,
jintArray wordArray) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
@ -210,7 +213,7 @@ static jint latinime_BinaryDictionary_getFrequency(JNIEnv *env, jobject object,
return dictionary->getFrequency(codePoints, codePointLength);
}
static jboolean latinime_BinaryDictionary_isValidBigram(JNIEnv *env, jobject object, jlong dict,
static jboolean latinime_BinaryDictionary_isValidBigram(JNIEnv *env, jclass clazz, jlong dict,
jintArray wordArray1, jintArray wordArray2) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return (jboolean) false;
@ -223,7 +226,7 @@ static jboolean latinime_BinaryDictionary_isValidBigram(JNIEnv *env, jobject obj
return dictionary->isValidBigram(codePoints1, codePointLength1, codePoints2, codePointLength2);
}
static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jobject object,
static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jclass clazz,
jintArray before, jintArray after, jint score) {
jsize beforeLength = env->GetArrayLength(before);
jsize afterLength = env->GetArrayLength(after);
@ -235,7 +238,7 @@ static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jobject
afterCodePoints, afterLength, score);
}
static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jobject object, jintArray before,
static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, jintArray before,
jintArray after) {
jsize beforeLength = env->GetArrayLength(before);
jsize afterLength = env->GetArrayLength(after);
@ -247,7 +250,7 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jobject object,
afterCodePoints, afterLength);
}
static void latinime_BinaryDictionary_close(JNIEnv *env, jobject object, jlong dict) {
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
const void *dictBuf = dictionary->getDict();
@ -277,7 +280,7 @@ static void releaseDictBuf(const void *dictBuf, const size_t length, const int f
}
static JNINativeMethod sMethods[] = {
{"openNative", "(Ljava/lang/String;JJI)J",
{"openNative", "(Ljava/lang/String;JJ)J",
reinterpret_cast<void *>(latinime_BinaryDictionary_open)},
{"closeNative", "(J)V", reinterpret_cast<void *>(latinime_BinaryDictionary_close)},
{"getSuggestionsNative", "(JJJ[I[I[I[I[IIIZ[IZ[I[I[I[I)I",

6
native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp
View file

@ -24,12 +24,12 @@
namespace latinime {
class Dictionary;
static jlong latinime_setDicTraverseSession(JNIEnv *env, jobject object, jstring localeJStr) {
static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr) {
void *traverseSession = DicTraverseWrapper::getDicTraverseSession(env, localeJStr);
return reinterpret_cast<jlong>(traverseSession);
}
static void latinime_initDicTraverseSession(JNIEnv *env, jobject object, jlong traverseSession,
static void latinime_initDicTraverseSession(JNIEnv *env, jclass clazz, jlong traverseSession,
jlong dictionary, jintArray previousWord, jint previousWordLength) {
void *ts = reinterpret_cast<void *>(traverseSession);
Dictionary *dict = reinterpret_cast<Dictionary *>(dictionary);
@ -42,7 +42,7 @@ static void latinime_initDicTraverseSession(JNIEnv *env, jobject object, jlong t
DicTraverseWrapper::initDicTraverseSession(ts, dict, prevWord, previousWordLength);
}
static void latinime_releaseDicTraverseSession(JNIEnv *env, jobject object, jlong traverseSession) {
static void latinime_releaseDicTraverseSession(JNIEnv *env, jclass clazz, jlong traverseSession) {
void *ts = reinterpret_cast<void *>(traverseSession);
DicTraverseWrapper::releaseDicTraverseSession(ts);
}

31
native/jni/src/bigram_dictionary.cpp
View file

@ -26,8 +26,7 @@
namespace latinime {
BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength)
: DICT(dict), MAX_WORD_LENGTH(maxWordLength) {
BigramDictionary::BigramDictionary(const uint8_t *const streamStart) : DICT_ROOT(streamStart) {
if (DEBUG_DICT) {
AKLOGI("BigramDictionary - constructor");
}
@ -84,8 +83,8 @@ void BigramDictionary::addWordBigram(int *word, int length, int frequency, int *
/* Parameters :
* prevWord: the word before, the one for which we need to look up bigrams.
* prevWordLength: its length.
* inputCodes: what user typed, in the same format as for UnigramDictionary::getSuggestions.
* codesSize: the size of the codes array.
* inputCodePoints: what user typed, in the same format as for UnigramDictionary::getSuggestions.
* inputSize: the size of the codes array.
* bigramCodePoints: an array for output, at the same format as outwords for getSuggestions.
* bigramFreq: an array to output frequencies.
* outputTypes: an array to output types.
@ -97,12 +96,12 @@ void BigramDictionary::addWordBigram(int *word, int length, int frequency, int *
* and the bigrams are used to boost unigram result scores, it makes little sense to
* reduce their scope to the ones that match the first letter.
*/
int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *inputCodes,
int codesSize, int *bigramCodePoints, int *bigramFreq, int *outputTypes) const {
int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *inputCodePoints,
int inputSize, int *bigramCodePoints, int *bigramFreq, int *outputTypes) const {
// TODO: remove unused arguments, and refrain from storing stuff in members of this class
// TODO: have "in" arguments before "out" ones, and make out args explicit in the name
const uint8_t *const root = DICT;
const uint8_t *const root = DICT_ROOT;
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
@ -124,8 +123,8 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i
const int length = BinaryFormat::getWordAtAddress(root, bigramPos, MAX_WORD_LENGTH,
bigramBuffer, &unigramFreq);
// codesSize == 0 means we are trying to find bigram predictions.
if (codesSize < 1 || checkFirstCharacter(bigramBuffer, inputCodes)) {
// inputSize == 0 means we are trying to find bigram predictions.
if (inputSize < 1 || checkFirstCharacter(bigramBuffer, inputCodePoints)) {
const int bigramFreqTemp = BinaryFormat::MASK_ATTRIBUTE_FREQUENCY & bigramFlags;
// Due to space constraints, the frequency for bigrams is approximate - the lower the
// unigram frequency, the worse the precision. The theoritical maximum error in
@ -147,7 +146,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i
int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const {
if (0 >= prevWordLength) return 0;
const uint8_t *const root = DICT;
const uint8_t *const root = DICT_ROOT;
int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength,
forceLowerCaseSearch);
@ -168,7 +167,7 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in
void BigramDictionary::fillBigramAddressToFrequencyMapAndFilter(const int *prevWord,
const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const {
memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE);
const uint8_t *const root = DICT;
const uint8_t *const root = DICT_ROOT;
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
if (0 == pos) {
@ -189,17 +188,17 @@ void BigramDictionary::fillBigramAddressToFrequencyMapAndFilter(const int *prevW
} while (0 != (BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags));
}
bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodes) const {
bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) const {
// Checks whether this word starts with same character or neighboring characters of
// what user typed.
int maxAlt = MAX_ALTERNATIVES;
const int firstBaseChar = toBaseLowerCase(*word);
const int firstBaseLowerCodePoint = toBaseLowerCase(*word);
while (maxAlt > 0) {
if (toBaseLowerCase(*inputCodes) == firstBaseChar) {
if (toBaseLowerCase(*inputCodePoints) == firstBaseLowerCodePoint) {
return true;
}
inputCodes++;
inputCodePoints++;
maxAlt--;
}
return false;
@ -207,7 +206,7 @@ bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodes) const {
bool BigramDictionary::isValidBigram(const int *word1, int length1, const int *word2,
int length2) const {
const uint8_t *const root = DICT;
const uint8_t *const root = DICT_ROOT;
int pos = getBigramListPositionForWord(word1, length1, false /* forceLowerCaseSearch */);
// getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
if (0 == pos) return false;

14
native/jni/src/bigram_dictionary.h
View file

@ -26,8 +26,8 @@ namespace latinime {
class BigramDictionary {
public:
BigramDictionary(const unsigned char *dict, int maxWordLength);
int getBigrams(const int *word, int length, int *inputCodes, int codesSize, int *outWords,
BigramDictionary(const uint8_t *const streamStart);
int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords,
int *frequencies, int *outputTypes) const;
void fillBigramAddressToFrequencyMapAndFilter(const int *prevWord, const int prevWordLength,
std::map<int, int> *map, uint8_t *filter) const;
@ -37,17 +37,11 @@ class BigramDictionary {
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary);
void addWordBigram(int *word, int length, int frequency, int *bigramFreq, int *bigramCodePoints,
int *outputTypes) const;
int getBigramAddress(int *pos, bool advance);
int getBigramFreq(int *pos);
void searchForTerminalNode(int addressLookingFor, int frequency);
bool getFirstBitOfByte(int *pos) { return (DICT[*pos] & 0x80) > 0; }
bool getSecondBitOfByte(int *pos) { return (DICT[*pos] & 0x40) > 0; }
bool checkFirstCharacter(int *word, int *inputCodes) const;
bool checkFirstCharacter(int *word, int *inputCodePoints) const;
int getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
const bool forceLowerCaseSearch) const;
const unsigned char *DICT;
const int MAX_WORD_LENGTH;
const uint8_t *const DICT_ROOT;
// TODO: Re-implement proximity correction for bigram correction
static const int MAX_ALTERNATIVES = 1;
};

4
native/jni/src/correction.cpp
View file

@ -30,7 +30,7 @@ class ProximityInfo;
/////////////////////////////
inline static void initEditDistance(int *editDistanceTable) {
for (int i = 0; i <= MAX_WORD_LENGTH_INTERNAL; ++i) {
for (int i = 0; i <= MAX_WORD_LENGTH; ++i) {
editDistanceTable[i] = i;
}
}
@ -77,7 +77,7 @@ void Correction::initCorrection(const ProximityInfo *pi, const int inputSize, co
mMaxDepth = maxDepth;
mMaxEditDistance = mInputSize < 5 ? 2 : mInputSize / 2;
// TODO: This is not supposed to be required. Check what's going wrong with
// editDistance[0 ~ MAX_WORD_LENGTH_INTERNAL]
// editDistance[0 ~ MAX_WORD_LENGTH]
initEditDistance(mEditDistanceTable);
}

12
native/jni/src/correction.h
View file

@ -57,7 +57,7 @@ class Correction {
// Non virtual inline destructor -- never inherit this class
~Correction() {}
void resetCorrection();
void initCorrection(const ProximityInfo *pi, const int inputSize, const int maxWordLength);
void initCorrection(const ProximityInfo *pi, const int inputSize, const int maxDepth);
void initCorrectionState(const int rootPos, const int childCount, const bool traverseAll);
// TODO: remove
@ -237,14 +237,14 @@ class Correction {
int mTotalTraverseCount;
// The following arrays are state buffer.
int mWord[MAX_WORD_LENGTH_INTERNAL];
int mDistances[MAX_WORD_LENGTH_INTERNAL];
int mWord[MAX_WORD_LENGTH];
int mDistances[MAX_WORD_LENGTH];
// Edit distance calculation requires a buffer with (N+1)^2 length for the input length N.
// Caveat: Do not create multiple tables per thread as this table eats up RAM a lot.
int mEditDistanceTable[(MAX_WORD_LENGTH_INTERNAL + 1) * (MAX_WORD_LENGTH_INTERNAL + 1)];
int mEditDistanceTable[(MAX_WORD_LENGTH + 1) * (MAX_WORD_LENGTH + 1)];
CorrectionState mCorrectionStates[MAX_WORD_LENGTH_INTERNAL];
CorrectionState mCorrectionStates[MAX_WORD_LENGTH];
// The following member variables are being used as cache values of the correction state.
bool mNeedsToTraverseAllNodes;
@ -336,7 +336,7 @@ inline Correction::CorrectionType Correction::processUnrelatedCorrectionType() {
AK_FORCE_INLINE static void calcEditDistanceOneStep(int *editDistanceTable, const int *input,
const int inputSize, const int *output, const int outputLength) {
// TODO: Make sure that editDistance[0 ~ MAX_WORD_LENGTH_INTERNAL] is not touched.
// TODO: Make sure that editDistance[0 ~ MAX_WORD_LENGTH] is not touched.
// Let dp[i][j] be editDistanceTable[i * (inputSize + 1) + j].
// Assuming that dp[0][0] ... dp[outputLength - 1][inputSize] are already calculated,
// and calculate dp[ouputLength][0] ... dp[outputLength][inputSize].

21
native/jni/src/defines.h
View file

@ -23,8 +23,10 @@
#define AK_FORCE_INLINE inline
#endif // __GNUC__
// This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java
#define MAX_WORD_LENGTH_INTERNAL 48
// Must be identical to Constants.Dictionary.MAX_WORD_LENGTH in Java
#define MAX_WORD_LENGTH 48
// Must be identical to BinaryDictionary.MAX_RESULTS in Java
#define MAX_RESULTS 18
#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG)
#include <android/log.h>
@ -34,8 +36,7 @@
#define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__)
#define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__)
#define DUMP_RESULT(words, frequencies, maxWordCount, maxWordLength) do { \
dumpResult(words, frequencies, maxWordCount, maxWordLength); } while (0)
#define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0)
#define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0)
#define INTS_TO_CHARS(input, length, output) do { \
intArrayToCharArray(input, length, output); } while (0)
@ -45,7 +46,7 @@ AK_FORCE_INLINE static int intArrayToCharArray(const int *source, const int sour
char *dest) {
int si = 0;
int di = 0;
while (si < sourceSize && di < MAX_WORD_LENGTH_INTERNAL - 1 && 0 != source[si]) {
while (si < sourceSize && di < MAX_WORD_LENGTH - 1 && 0 != source[si]) {
const int codePoint = source[si++];
if (codePoint < 0x7F) {
dest[di++] = codePoint;
@ -71,11 +72,10 @@ static inline void dumpWordInfo(const int *word, const int length, const int ran
}
}
static inline void dumpResult(const int *outWords, const int *frequencies, const int maxWordCounts,
const int maxWordLength) {
static inline void dumpResult(const int *outWords, const int *frequencies) {
AKLOGI("--- DUMP RESULT ---------");
for (int i = 0; i < maxWordCounts; ++i) {
dumpWordInfo(&outWords[i * maxWordLength], maxWordLength, i, frequencies[i]);
for (int i = 0; i < MAX_RESULTS; ++i) {
dumpWordInfo(&outWords[i * MAX_WORD_LENGTH], MAX_WORD_LENGTH, i, frequencies[i]);
}
AKLOGI("-------------------------");
}
@ -120,7 +120,7 @@ static inline void showStackTrace() {
#else
#define AKLOGE(fmt, ...)
#define AKLOGI(fmt, ...)
#define DUMP_RESULT(words, frequencies, maxWordCount, maxWordLength)
#define DUMP_RESULT(words, frequencies)
#define DUMP_WORD(word, length)
#undef DO_ASSERT_TEST
#define ASSERT(success)
@ -348,7 +348,6 @@ static inline void prof_out(void) {
#define MAX_DEPTH_MULTIPLIER 3
#define FIRST_WORD_INDEX 0
#define MAX_RESULTS 18 /* Must be identical to BinaryDictionary.MAX_RESULTS in Java */
// Max Distance between point to key
#define MAX_POINT_TO_KEY_LENGTH 10000000

44
native/jni/src/dictionary.cpp
View file

@ -28,21 +28,13 @@
namespace latinime {
Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int maxWordLength)
Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust)
: mDict(static_cast<unsigned char *>(dict)),
mOffsetDict((static_cast<unsigned char *>(dict)) + BinaryFormat::getHeaderSize(mDict)),
mDictSize(dictSize), mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust),
mUnigramDictionary(new UnigramDictionary(mOffsetDict, maxWordLength,
BinaryFormat::getFlags(mDict))),
mBigramDictionary(new BigramDictionary(mOffsetDict, maxWordLength)),
mGestureSuggest(new GestureSuggest(maxWordLength)) {
if (DEBUG_DICT) {
if (MAX_WORD_LENGTH_INTERNAL < maxWordLength) {
AKLOGI("Max word length (%d) is greater than %d",
maxWordLength, MAX_WORD_LENGTH_INTERNAL);
AKLOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF));
}
}
mUnigramDictionary(new UnigramDictionary(mOffsetDict, BinaryFormat::getFlags(mDict))),
mBigramDictionary(new BigramDictionary(mOffsetDict)),
mGestureSuggest(new GestureSuggest()) {
}
Dictionary::~Dictionary() {
@ -52,38 +44,38 @@ Dictionary::~Dictionary() {
}
int Dictionary::getSuggestions(ProximityInfo *proximityInfo, void *traverseSession,
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *codes,
int codesSize, int *prevWordChars, int prevWordLength, int commitPoint, bool isGesture,
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint, bool isGesture,
bool useFullEditDistance, int *outWords, int *frequencies, int *spaceIndices,
int *outputTypes) const {
int result = 0;
if (isGesture) {
DicTraverseWrapper::initDicTraverseSession(
traverseSession, this, prevWordChars, prevWordLength);
result = mGestureSuggest->getSuggestions(proximityInfo, traverseSession,
xcoordinates, ycoordinates, times, pointerIds, codes, codesSize, commitPoint,
outWords, frequencies, spaceIndices, outputTypes);
traverseSession, this, prevWordCodePoints, prevWordLength);
result = mGestureSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, outWords,
frequencies, spaceIndices, outputTypes);
if (DEBUG_DICT) {
DUMP_RESULT(outWords, frequencies, 18 /* MAX_WORDS */, MAX_WORD_LENGTH_INTERNAL);
DUMP_RESULT(outWords, frequencies);
}
return result;
} else {
std::map<int, int> bigramMap;
uint8_t bigramFilter[BIGRAM_FILTER_BYTE_SIZE];
mBigramDictionary->fillBigramAddressToFrequencyMapAndFilter(prevWordChars,
mBigramDictionary->fillBigramAddressToFrequencyMapAndFilter(prevWordCodePoints,
prevWordLength, &bigramMap, bigramFilter);
result = mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates,
ycoordinates, codes, codesSize, &bigramMap, bigramFilter,
useFullEditDistance, outWords, frequencies, outputTypes);
result = mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates,
inputCodePoints, inputSize, &bigramMap, bigramFilter, useFullEditDistance, outWords,
frequencies, outputTypes);
return result;
}
}
int Dictionary::getBigrams(const int *word, int length, int *codes, int codesSize,
int Dictionary::getBigrams(const int *word, int length, int *inputCodePoints, int inputSize,
int *outWords, int *frequencies, int *outputTypes) const {
if (length <= 0) return 0;
return mBigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies,
outputTypes);
return mBigramDictionary->getBigrams(word, length, inputCodePoints, inputSize, outWords,
frequencies, outputTypes);
}
int Dictionary::getFrequency(const int *word, int length) const {

8
native/jni/src/dictionary.h
View file

@ -41,15 +41,15 @@ class Dictionary {
const static int KIND_SHORTCUT = 7; // A shortcut
const static int KIND_PREDICTION = 8; // A prediction (== a suggestion with no input)
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int maxWordLength);
Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust);
int getSuggestions(ProximityInfo *proximityInfo, void *traverseSession, int *xcoordinates,
int *ycoordinates, int *times, int *pointerIds, int *codes, int codesSize,
int *prevWordChars, int prevWordLength, int commitPoint, bool isGesture,
int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, int inputSize,
int *prevWordCodePoints, int prevWordLength, int commitPoint, bool isGesture,
bool useFullEditDistance, int *outWords, int *frequencies, int *spaceIndices,
int *outputTypes) const;
int getBigrams(const int *word, int length, int *codes, int codesSize, int *outWords,
int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords,
int *frequencies, int *outputTypes) const;
int getFrequency(const int *word, int length) const;

4
native/jni/src/proximity_info_state.cpp
View file

@ -486,7 +486,7 @@ bool ProximityInfoState::checkAndReturnIsContinuationPossible(const int inputSiz
// Assuming the cache is invalid if the previous input size is larger than the new one.
return false;
}
for (int i = 0; i < mSampledInputSize && i < MAX_WORD_LENGTH_INTERNAL; ++i) {
for (int i = 0; i < mSampledInputSize && i < MAX_WORD_LENGTH; ++i) {
if (xCoordinates[i] != mSampledInputXs[i]
|| yCoordinates[i] != mSampledInputYs[i]) {
return false;
@ -1184,7 +1184,7 @@ float ProximityInfoState::getMostProbableString(int *const codePointBuf) const {
int index = 0;
float sumLogProbability = 0.0f;
// TODO: Current implementation is greedy algorithm. DP would be efficient for many cases.
for (int i = 0; i < mSampledInputSize && index < MAX_WORD_LENGTH_INTERNAL - 1; ++i) {
for (int i = 0; i < mSampledInputSize && index < MAX_WORD_LENGTH - 1; ++i) {
float minLogProbability = static_cast<float>(MAX_POINT_TO_KEY_LENGTH);
int character = NOT_AN_INDEX;
for (hash_map_compat<int, float>::const_iterator it = mCharProbabilities[i].begin();

6
native/jni/src/proximity_info_state.h
View file

@ -290,10 +290,10 @@ class ProximityInfoState {
// inputs including the current input point.
std::vector<NearKeycodesSet> mSearchKeysVector;
bool mTouchPositionCorrectionEnabled;
int mInputCodes[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH_INTERNAL];
int mNormalizedSquaredDistances[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH_INTERNAL];
int mInputCodes[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH];
int mNormalizedSquaredDistances[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH];
int mSampledInputSize;
int mPrimaryInputWord[MAX_WORD_LENGTH_INTERNAL];
int mPrimaryInputWord[MAX_WORD_LENGTH];
};
} // namespace latinime
#endif // LATINIME_PROXIMITY_INFO_STATE_H

2
native/jni/src/suggest/gesture_suggest.cpp
View file

@ -17,7 +17,7 @@
#include "gesture_suggest.h"
namespace latinime {
SuggestInterface *(*GestureSuggest::sGestureSuggestFactoryMethod)(int) = 0;
SuggestInterface *(*GestureSuggest::sGestureSuggestFactoryMethod)() = 0;
GestureSuggest::~GestureSuggest() {
delete mSuggestInterface;

14
native/jni/src/suggest/gesture_suggest.h
View file

@ -26,9 +26,7 @@ class ProximityInfo;
class GestureSuggest : public SuggestInterface {
public:
GestureSuggest(const int maxWordLength)
: mSuggestInterface(getGestureSuggestInstance(maxWordLength)) {
}
GestureSuggest() : mSuggestInterface(getGestureSuggestInstance()) {}
virtual ~GestureSuggest();
@ -43,20 +41,20 @@ class GestureSuggest : public SuggestInterface {
outputTypes);
}
static void setGestureSuggestFactoryMethod(SuggestInterface *(*factoryMethod)(int)) {
static void setGestureSuggestFactoryMethod(SuggestInterface *(*factoryMethod)()) {
sGestureSuggestFactoryMethod = factoryMethod;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(GestureSuggest);
static SuggestInterface *getGestureSuggestInstance(int maxWordLength) {
DISALLOW_COPY_AND_ASSIGN(GestureSuggest);
static SuggestInterface *getGestureSuggestInstance() {
if (!sGestureSuggestFactoryMethod) {
return 0;
}
return sGestureSuggestFactoryMethod(maxWordLength);
return sGestureSuggestFactoryMethod();
}
static SuggestInterface *(*sGestureSuggestFactoryMethod)(int);
static SuggestInterface *(*sGestureSuggestFactoryMethod)();
SuggestInterface *mSuggestInterface;
};
} // namespace latinime

2
native/jni/src/suggest/typing_suggest.cpp
View file

@ -17,7 +17,7 @@
#include "typing_suggest.h"
namespace latinime {
SuggestInterface *(*TypingSuggest::sTypingSuggestFactoryMethod)(int) = 0;
SuggestInterface *(*TypingSuggest::sTypingSuggestFactoryMethod)() = 0;
TypingSuggest::~TypingSuggest() {
delete mSuggestInterface;

13
native/jni/src/suggest/typing_suggest.h
View file

@ -26,8 +26,7 @@ class ProximityInfo;
class TypingSuggest : public SuggestInterface {
public:
TypingSuggest(const int maxWordLength)
: mSuggestInterface(getTypingSuggestInstance(maxWordLength)) {
TypingSuggest() : mSuggestInterface(getTypingSuggestInstance()) {
}
virtual ~TypingSuggest();
@ -43,20 +42,20 @@ class TypingSuggest : public SuggestInterface {
outputTypes);
}
static void setTypingSuggestFactoryMethod(SuggestInterface *(*factoryMethod)(int)) {
static void setTypingSuggestFactoryMethod(SuggestInterface *(*factoryMethod)()) {
sTypingSuggestFactoryMethod = factoryMethod;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(TypingSuggest);
static SuggestInterface *getTypingSuggestInstance(int maxWordLength) {
DISALLOW_COPY_AND_ASSIGN(TypingSuggest);
static SuggestInterface *getTypingSuggestInstance() {
if (!sTypingSuggestFactoryMethod) {
return 0;
}
return sTypingSuggestFactoryMethod(maxWordLength);
return sTypingSuggestFactoryMethod();
}
static SuggestInterface *(*sTypingSuggestFactoryMethod)(int);
static SuggestInterface *(*sTypingSuggestFactoryMethod)();
SuggestInterface *mSuggestInterface;
};
} // namespace latinime

11
native/jni/src/terminal_attributes.h
View file

@ -30,10 +30,6 @@ namespace latinime {
class TerminalAttributes {
public:
class ShortcutIterator {
const uint8_t *const mDict;
int mPos;
bool mHasNextShortcutTarget;
public:
ShortcutIterator(const uint8_t *dict, const int pos, const uint8_t flags)
: mDict(dict), mPos(pos),
@ -50,7 +46,7 @@ class TerminalAttributes {
const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos);
mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT);
unsigned int i;
for (i = 0; i < MAX_WORD_LENGTH_INTERNAL; ++i) {
for (i = 0; i < MAX_WORD_LENGTH; ++i) {
const int codePoint = BinaryFormat::getCodePointAndForwardPointer(mDict, &mPos);
if (NOT_A_CODE_POINT == codePoint) break;
outWord[i] = codePoint;
@ -58,6 +54,11 @@ class TerminalAttributes {
*outFreq = BinaryFormat::getAttributeFrequencyFromFlags(shortcutFlags);
return i;
}
private:
const uint8_t *const mDict;
int mPos;
bool mHasNextShortcutTarget;
};
TerminalAttributes(const uint8_t *const dict, const uint8_t flags, const int pos)

69
native/jni/src/unigram_dictionary.cpp
View file

@ -40,9 +40,8 @@ const UnigramDictionary::digraph_t UnigramDictionary::FRENCH_LIGATURES_DIGRAPHS[
{ 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE
// TODO: check the header
UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, int maxWordLength,
const unsigned int flags)
: DICT_ROOT(streamStart), MAX_WORD_LENGTH(maxWordLength), ROOT_POS(0),
UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, const unsigned int flags)
: DICT_ROOT(streamStart), ROOT_POS(0),
MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), FLAGS(flags) {
if (DEBUG_DICT) {
AKLOGI("UnigramDictionary - constructor");
@ -52,21 +51,17 @@ UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, int maxWo
UnigramDictionary::~UnigramDictionary() {
}
static inline int getCodesBufferSize(const int *codes, const int codesSize) {
return sizeof(*codes) * codesSize;
}
// TODO: This needs to take a const int* and not tinker with its contents
static void addWord(int *word, int length, int frequency, WordsPriorityQueue *queue, int type) {
queue->push(frequency, word, length, type);
}
// Return the replacement code point for a digraph, or 0 if none.
int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int codesSize,
int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int inputSize,
const digraph_t *const digraphs, const unsigned int digraphsSize) const {
// There can't be a digraph if we don't have at least 2 characters to examine
if (i + 2 > codesSize) return false;
if (i + 2 > inputSize) return false;
// Search for the first char of some digraph
int lastDigraphIndex = -1;
@ -87,7 +82,7 @@ int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, cons
// Mostly the same arguments as the non-recursive version, except:
// codes is the original value. It points to the start of the work buffer, and gets passed as is.
// codesSize is the size of the user input (thus, it is the size of codesSrc).
// inputSize is the size of the user input (thus, it is the size of codesSrc).
// codesDest is the current point in the work buffer.
// codesSrc is the current point in the user-input, original, content-unmodified buffer.
// codesRemain is the remaining size in codesSrc.
@ -167,49 +162,49 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit
// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter
// in bigram_dictionary.cpp
int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize,
const int *ycoordinates, const int *inputCodePoints, const int inputSize,
const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, int *outWords, int *frequencies, int *outputTypes) const {
WordsPriorityQueuePool queuePool(MAX_RESULTS, SUB_QUEUE_MAX_WORDS, MAX_WORD_LENGTH);
WordsPriorityQueuePool queuePool(MAX_RESULTS, SUB_QUEUE_MAX_WORDS);
queuePool.clearAll();
Correction masterCorrection;
masterCorrection.resetCorrection();
if (BinaryFormat::REQUIRES_GERMAN_UMLAUT_PROCESSING & FLAGS)
{ // Incrementally tune the word and try all possibilities
int codesBuffer[getCodesBufferSize(codes, codesSize)];
int xCoordinatesBuffer[codesSize];
int yCoordinatesBuffer[codesSize];
int codesBuffer[sizeof(*inputCodePoints) * inputSize];
int xCoordinatesBuffer[inputSize];
int yCoordinatesBuffer[inputSize];
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection,
xCoordinatesBuffer, yCoordinatesBuffer, inputSize, bigramMap, bigramFilter,
useFullEditDistance, inputCodePoints, inputSize, 0, codesBuffer, &masterCorrection,
&queuePool, GERMAN_UMLAUT_DIGRAPHS, NELEMS(GERMAN_UMLAUT_DIGRAPHS));
} else if (BinaryFormat::REQUIRES_FRENCH_LIGATURES_PROCESSING & FLAGS) {
int codesBuffer[getCodesBufferSize(codes, codesSize)];
int xCoordinatesBuffer[codesSize];
int yCoordinatesBuffer[codesSize];
int codesBuffer[sizeof(*inputCodePoints) * inputSize];
int xCoordinatesBuffer[inputSize];
int yCoordinatesBuffer[inputSize];
getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer,
xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter,
useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection,
xCoordinatesBuffer, yCoordinatesBuffer, inputSize, bigramMap, bigramFilter,
useFullEditDistance, inputCodePoints, inputSize, 0, codesBuffer, &masterCorrection,
&queuePool, FRENCH_LIGATURES_DIGRAPHS, NELEMS(FRENCH_LIGATURES_DIGRAPHS));
} else { // Normal processing
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize,
getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, inputSize,
bigramMap, bigramFilter, useFullEditDistance, &masterCorrection, &queuePool);
}
PROF_START(20);
if (DEBUG_DICT) {
float ns = queuePool.getMasterQueue()->getHighestNormalizedScore(
masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0);
masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0);
ns += 0;
AKLOGI("Max normalized score = %f", ns);
}
const int suggestedWordsCount =
queuePool.getMasterQueue()->outputSuggestions(masterCorrection.getPrimaryInputWord(),
codesSize, frequencies, outWords, outputTypes);
inputSize, frequencies, outWords, outputTypes);
if (DEBUG_DICT) {
float ns = queuePool.getMasterQueue()->getHighestNormalizedScore(
masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0);
masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0);
ns += 0;
AKLOGI("Returning %d words", suggestedWordsCount);
/// Print the returned words
@ -227,7 +222,7 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *x
}
void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int inputSize,
const int *ycoordinates, const int *inputCodePoints, const int inputSize,
const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool)
const {
@ -236,8 +231,8 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const i
PROF_END(0);
PROF_START(1);
getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, bigramMap, bigramFilter,
useFullEditDistance, inputSize, correction, queuePool);
getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, bigramMap,
bigramFilter, useFullEditDistance, inputSize, correction, queuePool);
PROF_END(1);
PROF_START(2);
@ -262,7 +257,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const i
// Multiple word suggestions
if (SUGGEST_MULTIPLE_WORDS
&& inputSize >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) {
getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints,
useFullEditDistance, inputSize, correction, queuePool,
hasAutoCorrectionCandidate);
}
@ -398,10 +393,10 @@ void UnigramDictionary::onTerminal(const int probability,
// so that the insert order is protected inside the queue for words
// with the same score. For the moment we use -1 to make sure the shortcut will
// never be in front of the word.
int shortcutTarget[MAX_WORD_LENGTH_INTERNAL];
int shortcutTarget[MAX_WORD_LENGTH];
int shortcutFrequency;
const int shortcutTargetStringLength = iterator.getNextShortcutTarget(
MAX_WORD_LENGTH_INTERNAL, shortcutTarget, &shortcutFrequency);
MAX_WORD_LENGTH, shortcutTarget, &shortcutFrequency);
int shortcutScore;
int kind;
if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_FREQUENCY
@ -487,7 +482,7 @@ int UnigramDictionary::getSubStringSuggestion(
initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
inputSize, correction);
int word[MAX_WORD_LENGTH_INTERNAL];
int word[MAX_WORD_LENGTH];
int freq = getMostFrequentWordLike(
inputWordStartPos, inputWordLength, correction, word);
if (freq > 0) {
@ -761,13 +756,13 @@ static inline void onTerminalWordLike(const int freq, int *newWord, const int le
// that is, everything that only differs by case/accents.
int UnigramDictionary::getMostFrequentWordLikeInner(const int *const inWord, const int inputSize,
int *outWord) const {
int newWord[MAX_WORD_LENGTH_INTERNAL];
int newWord[MAX_WORD_LENGTH];
int depth = 0;
int maxFreq = -1;
const uint8_t *const root = DICT_ROOT;
int stackChildCount[MAX_WORD_LENGTH_INTERNAL];
int stackInputIndex[MAX_WORD_LENGTH_INTERNAL];
int stackSiblingPos[MAX_WORD_LENGTH_INTERNAL];
int stackChildCount[MAX_WORD_LENGTH];
int stackInputIndex[MAX_WORD_LENGTH];
int stackSiblingPos[MAX_WORD_LENGTH];
int startPos = 0;
stackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos);

12
native/jni/src/unigram_dictionary.h
View file

@ -39,12 +39,11 @@ class UnigramDictionary {
static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0;
static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1;
static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2;
UnigramDictionary(const uint8_t *const streamStart, int maxWordLength,
const unsigned int flags);
UnigramDictionary(const uint8_t *const streamStart, const unsigned int flags);
int getFrequency(const int *const inWord, const int length) const;
int getBigramPosition(int pos, int *word, int offset, int length) const;
int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize,
const int *ycoordinates, const int *inputCodePoints, const int inputSize,
const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, int *outWords, int *frequencies,
int *outputTypes) const;
@ -53,11 +52,11 @@ class UnigramDictionary {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary);
void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int inputSize,
const int *ycoordinates, const int *inputCodePoints, const int inputSize,
const std::map<int, int> *bigramMap, const uint8_t *bigramFilter,
const bool useFullEditDistance, Correction *correction,
WordsPriorityQueuePool *queuePool) const;
int getDigraphReplacement(const int *codes, const int i, const int codesSize,
int getDigraphReplacement(const int *codes, const int i, const int inputSize,
const digraph_t *const digraphs, const unsigned int digraphsSize) const;
void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codesBuffer, int *xCoordinatesBuffer,
@ -67,7 +66,7 @@ class UnigramDictionary {
WordsPriorityQueuePool *queuePool, const digraph_t *const digraphs,
const unsigned int digraphsSize) const;
void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize,
const int *ycoordinates, const int *codes, const int inputSize,
Correction *correction) const;
void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const std::map<int, int> *bigramMap,
@ -109,7 +108,6 @@ class UnigramDictionary {
int *outputWord) const;
const uint8_t *const DICT_ROOT;
const int MAX_WORD_LENGTH;
const int ROOT_POS;
const int MAX_DIGRAPH_SEARCH_DEPTH;
const int FLAGS;

15
native/jni/src/words_priority_queue.h
View file

@ -27,10 +27,9 @@ namespace latinime {
class WordsPriorityQueue {
public:
class SuggestedWord {
public:
struct SuggestedWord {
int mScore;
int mWord[MAX_WORD_LENGTH_INTERNAL];
int mWord[MAX_WORD_LENGTH];
int mWordLength;
bool mUsed;
int mType;
@ -44,11 +43,10 @@ class WordsPriorityQueue {
}
};
WordsPriorityQueue(int maxWords, int maxWordLength)
: mSuggestions(), MAX_WORDS(static_cast<unsigned int>(maxWords)),
MAX_WORD_LENGTH(static_cast<unsigned int>(maxWordLength)),
mSuggestedWords(new SuggestedWord[maxWordLength]), mHighestSuggestedWord(0) {
for (int i = 0; i < maxWordLength; ++i) {
WordsPriorityQueue(int maxWords)
: mSuggestions(), MAX_WORDS(maxWords),
mSuggestedWords(new SuggestedWord[MAX_WORD_LENGTH]), mHighestSuggestedWord(0) {
for (int i = 0; i < MAX_WORD_LENGTH; ++i) {
mSuggestedWords[i].mUsed = false;
}
}
@ -171,7 +169,6 @@ class WordsPriorityQueue {
wordComparator> Suggestions;
Suggestions mSuggestions;
const int MAX_WORDS;
const int MAX_WORD_LENGTH;
SuggestedWord *mSuggestedWords;
SuggestedWord *mHighestSuggestedWord;
};

7
native/jni/src/words_priority_queue_pool.h
View file

@ -24,15 +24,14 @@ namespace latinime {
class WordsPriorityQueuePool {
public:
WordsPriorityQueuePool(int mainQueueMaxWords, int subQueueMaxWords, int maxWordLength)
WordsPriorityQueuePool(int mainQueueMaxWords, int subQueueMaxWords)
// Note: using placement new() requires the caller to call the destructor explicitly.
: mMasterQueue(new(mMasterQueueBuf) WordsPriorityQueue(
mainQueueMaxWords, maxWordLength)) {
: mMasterQueue(new(mMasterQueueBuf) WordsPriorityQueue(mainQueueMaxWords)) {
for (int i = 0, subQueueBufOffset = 0;
i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT;
++i, subQueueBufOffset += static_cast<int>(sizeof(WordsPriorityQueue))) {
mSubQueues[i] = new(mSubQueueBuf + subQueueBufOffset)
WordsPriorityQueue(subQueueMaxWords, maxWordLength);
WordsPriorityQueue(subQueueMaxWords);
}
}