nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am a0befc64: Start passing timestamp for dynamic dictionaries. 

* commit 'a0befc6490ad295455782a7b6dbe8785788299e4':
  Start passing timestamp for dynamic dictionaries.
main
Keisuke Kuroyanagi 2013-12-03 19:39:55 -08:00 committed by Android Git Automerger
parent 35b0b0391f a0befc6490
commit f88de35725
8 changed files with 68 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -57,7 +57,7 @@ public final class BinaryDictionary extends Dictionary {
@UsedForTesting
public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT";
private static final int NOT_A_VALID_TIME_STAMP = -1;
public static final int NOT_A_VALID_TIME_STAMP = -1;
private long mNativeDict;
private final Locale mLocale;
@ -138,9 +138,9 @@ public final class BinaryDictionary extends Dictionary {
private static native int editDistanceNative(int[] before, int[] after);
private static native void addUnigramWordNative(long dict, int[] word, int probability,
int[] shortcutTarget, int shortcutProbability, boolean isNotAWord,
boolean isBlacklisted, int timeStamp);
boolean isBlacklisted, int timestamp);
private static native void addBigramWordsNative(long dict, int[] word0, int[] word1,
int probability, int timeStamp);
int probability, int timestamp);
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
private static native int addMultipleDictionaryEntriesNative(long dict,
LanguageModelParam[] languageModelParams, int startIndex);
@ -291,12 +291,26 @@ public final class BinaryDictionary extends Dictionary {
return;
}
final int[] codePoints = StringUtils.toCodePointArray(word);
final int[] shortcutTarget = new int[0];
addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTarget,
final int[] shortcutTargetCodePoints = new int[0];
addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints,
NOT_A_PROBABILITY, false /* isNotAWord */, false /* isBlacklisted */,
NOT_A_VALID_TIME_STAMP);
}
// Add a unigram entry to binary dictionary with unigram attributes in native code.
public void addUnigramWord(final String word, final int probability,
final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
if (TextUtils.isEmpty(word)) {
return;
}
final int[] codePoints = StringUtils.toCodePointArray(word);
final int[] shortcutTargetCodePoints = (shortcutTarget != null) ?
StringUtils.toCodePointArray(shortcutTarget) : null;
addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints,
shortcutProbability, isNotAWord, isBlacklisted, timestamp);
}
// Add a bigram entry to binary dictionary in native code.
public void addBigramWords(final String word0, final String word1, final int probability) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
@ -308,6 +322,17 @@ public final class BinaryDictionary extends Dictionary {
NOT_A_VALID_TIME_STAMP);
}
// Add a bigram entry to binary dictionary with timestamp in native code.
public void addBigramWords(final String word0, final String word1, final int probability,
final int timestamp) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp);
}
// Remove a bigram entry form binary dictionary in native code.
public void removeBigramWords(final String word0, final String word1) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
@ -327,10 +352,11 @@ public final class BinaryDictionary extends Dictionary {
public final int mShortcutProbability;
public final boolean mIsNotAWord;
public final boolean mIsBlacklisted;
public final int mTimeStamp;
public final int mTimestamp;
// Constructor for unigram.
public LanguageModelParam(final String word, final int unigramProbability) {
public LanguageModelParam(final String word, final int unigramProbability,
final int timestamp) {
mWord0 = null;
mWord1 = StringUtils.toCodePointArray(word);
mShortcutTarget = null;
@ -339,12 +365,13 @@ public final class BinaryDictionary extends Dictionary {
mShortcutProbability = NOT_A_PROBABILITY;
mIsNotAWord = false;
mIsBlacklisted = false;
mTimeStamp = NOT_A_VALID_TIME_STAMP;
mTimestamp = timestamp;
}
// Constructor for unigram and bigram.
public LanguageModelParam(final String word0, final String word1,
final int unigramProbability, final int bigramProbability) {
final int unigramProbability, final int bigramProbability,
final int timestamp) {
mWord0 = StringUtils.toCodePointArray(word0);
mWord1 = StringUtils.toCodePointArray(word1);
mShortcutTarget = null;
@ -353,7 +380,7 @@ public final class BinaryDictionary extends Dictionary {
mShortcutProbability = NOT_A_PROBABILITY;
mIsNotAWord = false;
mIsBlacklisted = false;
mTimeStamp = NOT_A_VALID_TIME_STAMP;
mTimestamp = timestamp;
}
}

12
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -334,8 +334,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Dynamically adds a word unigram to the dictionary. May overwrite an existing entry.
*/
protected void addWordDynamically(final String word, final String shortcutTarget,
final int frequency, final int shortcutFreq, final boolean isNotAWord) {
protected void addWordDynamically(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
if (!mIsUpdatable) {
Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename);
return;
@ -344,7 +345,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@Override
public void run() {
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
mBinaryDictionary.addUnigramWord(word, frequency);
mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp);
}
});
}
@ -353,7 +355,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
* Dynamically adds a word bigram in the dictionary. May overwrite an existing entry.
*/
protected void addBigramDynamically(final String word0, final String word1,
final int frequency) {
final int frequency, final int timestamp) {
if (!mIsUpdatable) {
Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: "
+ mFilename);
@ -363,7 +365,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@Override
public void run() {
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
mBinaryDictionary.addBigramWords(word0, word1, frequency);
mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp);
}
});
}

4
java/src/com/android/inputmethod/latin/LatinIME.java
View File

@ -109,6 +109,7 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Locale;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
/**
* Input method implementation for Qwerty'ish keyboard.
@ -2871,7 +2872,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final int maxFreq = AutoCorrectionUtils.getMaxFrequency(
suggest.getUnigramDictionaries(), suggestion);
if (maxFreq == 0) return null;
userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0);
userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0,
(int)TimeUnit.MILLISECONDS.toSeconds((System.currentTimeMillis())));
return prevWord;
}

9
java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
View File

@ -141,21 +141,22 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
public void addToDictionary(final String word0, final String word1, final boolean isValid) {
public void addToDictionary(final String word0, final String word1, final boolean isValid,
final int timestamp) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
}
final int frequency = isValid ?
FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
false /* isNotAWord */);
addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */,
false /* isNotAWord */, false /* isBlacklisted */, timestamp);
// Do not insert a word as a bigram of itself
if (word1.equals(word0)) {
return;
}
if (null != word0) {
addBigramDynamically(word0, word1, frequency);
addBigramDynamically(word0, word1, frequency, timestamp);
}
}

4
java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
View File

@ -90,12 +90,12 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Support multi locale to add bigram
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
int frequency) {
int frequency, int timestamp) {
final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) {
return;
}
dictionary.addToDictionary(word0, word1, isValid);
dictionary.addToDictionary(word0, word1, isValid, timestamp);
}
// TODO: Support multi locale.

18
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -284,7 +284,7 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji
static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict,
jintArray word, jint probability, jintArray shortcutTarget, jint shortuctProbability,
jboolean isNotAWord, jboolean isBlacklisted, jint timeStamp) {
jboolean isNotAWord, jboolean isBlacklisted, jint timestamp) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) {
return;
@ -297,11 +297,11 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz,
if (shortcutTarget) {
env->GetIntArrayRegion(shortcutTarget, 0, shortcutLength, shortcutTargetCodePoints);
}
dictionary->addUnigramWord(codePoints, wordLength, probability, timeStamp);
dictionary->addUnigramWord(codePoints, wordLength, probability, timestamp);
}
static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict,
jintArray word0, jintArray word1, jint probability, jint timeStamp) {
jintArray word0, jintArray word1, jint probability, jint timestamp) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) {
return;
@ -313,7 +313,7 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz,
int word1CodePoints[word1Length];
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints,
word1Length, probability, timeStamp);
word1Length, probability, timestamp);
}
static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict,
@ -355,8 +355,8 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
env->GetFieldID(languageModelParamClass, "mUnigramProbability", "I");
jfieldID bigramProbabilityFieldId =
env->GetFieldID(languageModelParamClass, "mBigramProbability", "I");
jfieldID timeStampFieldId =
env->GetFieldID(languageModelParamClass, "mTimeStamp", "I");
jfieldID timestampFieldId =
env->GetFieldID(languageModelParamClass, "mTimestamp", "I");
env->DeleteLocalRef(languageModelParamClass);
for (int i = startIndex; i < languageModelParamCount; ++i) {
@ -377,12 +377,12 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
int word1CodePoints[word1Length];
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
jint unigramProbability = env->GetIntField(languageModelParam, unigramProbabilityFieldId);
jint timeStamp = env->GetIntField(languageModelParam, timeStampFieldId);
dictionary->addUnigramWord(word1CodePoints, word1Length, unigramProbability, timeStamp);
jint timestamp = env->GetIntField(languageModelParam, timestampFieldId);
dictionary->addUnigramWord(word1CodePoints, word1Length, unigramProbability, timestamp);
if (word0) {
jint bigramProbability = env->GetIntField(languageModelParam, bigramProbabilityFieldId);
dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length,
bigramProbability, timeStamp);
bigramProbability, timestamp);
}
if (dictionary->needsToRunGC(true /* mindsBlockByGC */)) {
return i + 1;

5
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -780,10 +780,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = random.nextInt(0xF);
unigramProbabilities.put(word, probability);
if (prevWord == null) {
languageModelParams[i] = new LanguageModelParam(word, probability);
languageModelParams[i] = new LanguageModelParam(word, probability,
BinaryDictionary.NOT_A_VALID_TIME_STAMP);
} else {
languageModelParams[i] = new LanguageModelParam(prevWord, word, probability,
bigramProbability);
bigramProbability, BinaryDictionary.NOT_A_VALID_TIME_STAMP);
bigramProbabilities.put(new Pair<String, String>(prevWord, word),
bigramProbability);
}

3
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -71,7 +71,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
private void addToDict(final UserHistoryDictionary dict, final List<String> words) {
String prevWord = null;
for (String word : words) {
dict.addToDictionary(prevWord, word, true);
// TODO: Use timestamp properly.
dict.addToDictionary(prevWord, word, true, 0 /* timestamp */);
prevWord = word;
}
}