nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Use PrevWordsInfo for get/add/remove n-gram(bigram) entry."

main
Keisuke Kuroyanagi 2014-05-20 05:36:10 +00:00 committed by Android (Google) Code Review
parent 5b7c85ec32 ff50b39176
commit 1afaf6770b
15 changed files with 246 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -359,14 +359,16 @@ public final class BinaryDictionary extends Dictionary {
} }
@UsedForTesting @UsedForTesting
public boolean isValidBigram(final String word0, final String word1) { public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) {
return getBigramProbability(word0, word1) != NOT_A_PROBABILITY; return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY;
} }
public int getBigramProbability(final String word0, final String word1) { public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return NOT_A_PROBABILITY; if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
final int[] codePoints0 = StringUtils.toCodePointArray(word0); return NOT_A_PROBABILITY;
final int[] codePoints1 = StringUtils.toCodePointArray(word1); }
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word);
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
} }
@ -417,7 +419,7 @@ public final class BinaryDictionary extends Dictionary {
} }
// Add a unigram entry to binary dictionary with unigram attributes in native code. // Add a unigram entry to binary dictionary with unigram attributes in native code.
public void addUnigramWord(final String word, final int probability, public void addUnigramEntry(final String word, final int probability,
final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) { final boolean isBlacklisted, final int timestamp) {
if (TextUtils.isEmpty(word)) { if (TextUtils.isEmpty(word)) {
@ -431,25 +433,26 @@ public final class BinaryDictionary extends Dictionary {
mHasUpdated = true; mHasUpdated = true;
} }
// Add a bigram entry to binary dictionary with timestamp in native code. // Add an n-gram entry to the binary dictionary with timestamp in native code.
public void addBigramWords(final String word0, final String word1, final int probability, public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
final int probability,
final int timestamp) { final int timestamp) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
return; return;
} }
final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word1); final int[] codePoints1 = StringUtils.toCodePointArray(word);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp); addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp);
mHasUpdated = true; mHasUpdated = true;
} }
// Remove a bigram entry form binary dictionary in native code. // Remove an n-gram entry from the binary dictionary in native code.
public void removeBigramWords(final String word0, final String word1) { public void removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
return; return;
} }
final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word1); final int[] codePoints1 = StringUtils.toCodePointArray(word);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
mHasUpdated = true; mHasUpdated = true;
} }

16
java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
View File

@ -142,7 +142,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
Log.d(TAG, "loadAccountVocabulary: " + word); Log.d(TAG, "loadAccountVocabulary: " + word);
} }
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP); BinaryDictionary.NOT_A_VALID_TIMESTAMP);
} }
@ -224,7 +224,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
*/ */
private void addNameLocked(final String name) { private void addNameLocked(final String name) {
int len = StringUtils.codePointCount(name); int len = StringUtils.codePointCount(name);
String prevWord = null; PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
// TODO: Better tokenization for non-Latin writing systems // TODO: Better tokenization for non-Latin writing systems
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
if (Character.isLetter(name.codePointAt(i))) { if (Character.isLetter(name.codePointAt(i))) {
@ -239,19 +239,19 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
final int wordLen = StringUtils.codePointCount(word); final int wordLen = StringUtils.codePointCount(word);
if (wordLen < MAX_WORD_LENGTH && wordLen > 1) { if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
if (DEBUG) { if (DEBUG) {
Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord); Log.d(TAG, "addName " + name + ", " + word + ", "
+ prevWordsInfo.mPrevWord);
} }
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, addUnigramLocked(word, FREQUENCY_FOR_CONTACTS,
null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && mUseFirstLastBigrams) {
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addBigramDynamicallyLocked(prevWord, word, addNgramEntryLocked(prevWordsInfo, word, FREQUENCY_FOR_CONTACTS_BIGRAM,
FREQUENCY_FOR_CONTACTS_BIGRAM,
BinaryDictionary.NOT_A_VALID_TIMESTAMP); BinaryDictionary.NOT_A_VALID_TIMESTAMP);
} }
prevWord = word; prevWordsInfo = new PrevWordsInfo(word);
} }
} }
} }

20
java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
View File

@ -370,22 +370,23 @@ public class DictionaryFacilitatorForSuggest {
} }
public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized,
final String previousWord, final int timeStampInSeconds, final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds,
final boolean blockPotentiallyOffensive) { final boolean blockPotentiallyOffensive) {
final Dictionaries dictionaries = mDictionaries; final Dictionaries dictionaries = mDictionaries;
final String[] words = suggestion.split(Constants.WORD_SEPARATOR); final String[] words = suggestion.split(Constants.WORD_SEPARATOR);
for (int i = 0; i < words.length; i++) { for (int i = 0; i < words.length; i++) {
final String currentWord = words[i]; final String currentWord = words[i];
final String prevWord = (i == 0) ? previousWord : words[i - 1]; final PrevWordsInfo prevWordsInfoForCurrentWord =
(i == 0) ? prevWordsInfo : new PrevWordsInfo(words[i - 1]);
final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false; final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false;
addWordToUserHistory(dictionaries, prevWord, currentWord, addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord,
wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive); wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive);
} }
} }
private void addWordToUserHistory(final Dictionaries dictionaries, final String prevWord, private void addWordToUserHistory(final Dictionaries dictionaries,
final String word, final boolean wasAutoCapitalized, final int timeStampInSeconds, final PrevWordsInfo prevWordsInfo, final String word, final boolean wasAutoCapitalized,
final boolean blockPotentiallyOffensive) { final int timeStampInSeconds, final boolean blockPotentiallyOffensive) {
final ExpandableBinaryDictionary userHistoryDictionary = final ExpandableBinaryDictionary userHistoryDictionary =
dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary == null) { if (userHistoryDictionary == null) {
@ -430,15 +431,16 @@ public class DictionaryFacilitatorForSuggest {
// We demote unrecognized words (frequency < 0, below) by specifying them as "invalid". // We demote unrecognized words (frequency < 0, below) by specifying them as "invalid".
// We don't add words with 0-frequency (assuming they would be profanity etc.). // We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0; final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWord, secondWord, UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord,
isValid, timeStampInSeconds); isValid, timeStampInSeconds);
} }
public void cancelAddingUserHistory(final String previousWord, final String committedWord) { public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo,
final String committedWord) {
final ExpandableBinaryDictionary userHistoryDictionary = final ExpandableBinaryDictionary userHistoryDictionary =
mDictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); mDictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary != null) { if (userHistoryDictionary != null) {
userHistoryDictionary.removeBigramDynamically(previousWord, committedWord); userHistoryDictionary.removeNgramDynamically(prevWordsInfo, committedWord);
} }
} }

30
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -269,9 +269,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
} }
/** /**
* Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. * Adds unigram information of a word to the dictionary. May overwrite an existing entry.
*/ */
public void addWordDynamically(final String word, final int frequency, public void addUnigramEntry(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) { final boolean isBlacklisted, final int timestamp) {
reloadDictionaryIfRequired(); reloadDictionaryIfRequired();
@ -282,23 +282,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return; return;
} }
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp); isNotAWord, isBlacklisted, timestamp);
} }
}); });
} }
protected void addWordDynamicallyLocked(final String word, final int frequency, protected void addUnigramLocked(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) { final boolean isBlacklisted, final int timestamp) {
mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp); isNotAWord, isBlacklisted, timestamp);
} }
/** /**
* Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. * Adds n-gram information of a word to the dictionary. May overwrite an existing entry.
*/ */
public void addBigramDynamically(final String word0, final String word1, public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
final int frequency, final int timestamp) { final int frequency, final int timestamp) {
reloadDictionaryIfRequired(); reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() { asyncExecuteTaskWithWriteLock(new Runnable() {
@ -308,20 +308,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return; return;
} }
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addBigramDynamicallyLocked(word0, word1, frequency, timestamp); addNgramEntryLocked(prevWordsInfo, word, frequency, timestamp);
} }
}); });
} }
protected void addBigramDynamicallyLocked(final String word0, final String word1, protected void addNgramEntryLocked(final PrevWordsInfo prevWordsInfo, final String word,
final int frequency, final int timestamp) { final int frequency, final int timestamp) {
mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); mBinaryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp);
} }
/** /**
* Dynamically remove a word bigram in the dictionary. * Dynamically remove the n-gram entry in the dictionary.
*/ */
public void removeBigramDynamically(final String word0, final String word1) { public void removeNgramDynamically(final PrevWordsInfo prevWordsInfo, final String word1) {
reloadDictionaryIfRequired(); reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() { asyncExecuteTaskWithWriteLock(new Runnable() {
@Override @Override
@ -330,7 +330,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return; return;
} }
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
mBinaryDictionary.removeBigramWords(word0, word1); mBinaryDictionary.removeNgramEntry(prevWordsInfo, word1);
} }
}); });
} }
@ -428,9 +428,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return mBinaryDictionary.isValidWord(word); return mBinaryDictionary.isValidWord(word);
} }
protected boolean isValidBigramLocked(final String word1, final String word2) { protected boolean isValidNgramLocked(final PrevWordsInfo prevWordsInfo, final String word) {
if (mBinaryDictionary == null) return false; if (mBinaryDictionary == null) return false;
return mBinaryDictionary.isValidBigram(word1, word2); return mBinaryDictionary.isValidNgram(prevWordsInfo, word);
} }
/** /**

6
java/src/com/android/inputmethod/latin/LastComposedWord.java
View File

@ -48,7 +48,7 @@ public final class LastComposedWord {
public final String mTypedWord; public final String mTypedWord;
public final CharSequence mCommittedWord; public final CharSequence mCommittedWord;
public final String mSeparatorString; public final String mSeparatorString;
public final String mPrevWord; public final PrevWordsInfo mPrevWordsInfo;
public final int mCapitalizedMode; public final int mCapitalizedMode;
public final InputPointers mInputPointers = public final InputPointers mInputPointers =
new InputPointers(Constants.DICTIONARY_MAX_WORD_LENGTH); new InputPointers(Constants.DICTIONARY_MAX_WORD_LENGTH);
@ -64,7 +64,7 @@ public final class LastComposedWord {
public LastComposedWord(final ArrayList<Event> events, public LastComposedWord(final ArrayList<Event> events,
final InputPointers inputPointers, final String typedWord, final InputPointers inputPointers, final String typedWord,
final CharSequence committedWord, final String separatorString, final CharSequence committedWord, final String separatorString,
final String prevWord, final int capitalizedMode) { final PrevWordsInfo prevWordsInfo, final int capitalizedMode) {
if (inputPointers != null) { if (inputPointers != null) {
mInputPointers.copy(inputPointers); mInputPointers.copy(inputPointers);
} }
@ -73,7 +73,7 @@ public final class LastComposedWord {
mCommittedWord = committedWord; mCommittedWord = committedWord;
mSeparatorString = separatorString; mSeparatorString = separatorString;
mActive = true; mActive = true;
mPrevWord = prevWord; mPrevWordsInfo = prevWordsInfo;
mCapitalizedMode = capitalizedMode; mCapitalizedMode = capitalizedMode;
} }

4
java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
View File

@ -258,12 +258,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
// Safeguard against adding really long words. // Safeguard against adding really long words.
if (word.length() < MAX_WORD_LENGTH) { if (word.length() < MAX_WORD_LENGTH) {
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */, addUnigramLocked(word, adjustedFrequency, null /* shortcutTarget */,
0 /* shortcutFreq */, false /* isNotAWord */, 0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addWordDynamicallyLocked(shortcut, adjustedFrequency, word, addUnigramLocked(shortcut, adjustedFrequency, word,
USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */, USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
} }

11
java/src/com/android/inputmethod/latin/WordComposer.java
View File

@ -294,11 +294,10 @@ public final class WordComposer {
* This will register NOT_A_COORDINATE for X and Ys, and use the passed keyboard for proximity. * This will register NOT_A_COORDINATE for X and Ys, and use the passed keyboard for proximity.
* @param codePoints the code points to set as the composing word. * @param codePoints the code points to set as the composing word.
* @param coordinates the x, y coordinates of the key in the CoordinateUtils format * @param coordinates the x, y coordinates of the key in the CoordinateUtils format
* @param previousWord the previous word, to use as context for suggestions. Can be null if * @param prevWordsInfo the information of previous words, to use as context for suggestions
* the context is nil (typically, at start of text).
*/ */
public void setComposingWord(final int[] codePoints, final int[] coordinates, public void setComposingWord(final int[] codePoints, final int[] coordinates,
final CharSequence previousWord) { final PrevWordsInfo prevWordsInfo) {
reset(); reset();
final int length = codePoints.length; final int length = codePoints.length;
for (int i = 0; i < length; ++i) { for (int i = 0; i < length; ++i) {
@ -307,7 +306,7 @@ public final class WordComposer {
CoordinateUtils.yFromArray(coordinates, i))); CoordinateUtils.yFromArray(coordinates, i)));
} }
mIsResumed = true; mIsResumed = true;
mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString()); mPrevWordsInfo = prevWordsInfo;
} }
/** /**
@ -413,13 +412,13 @@ public final class WordComposer {
// `type' should be one of the LastComposedWord.COMMIT_TYPE_* constants above. // `type' should be one of the LastComposedWord.COMMIT_TYPE_* constants above.
// committedWord should contain suggestion spans if applicable. // committedWord should contain suggestion spans if applicable.
public LastComposedWord commitWord(final int type, final CharSequence committedWord, public LastComposedWord commitWord(final int type, final CharSequence committedWord,
final String separatorString, final String prevWord) { final String separatorString, final PrevWordsInfo prevWordsInfo) {
// Note: currently, we come here whenever we commit a word. If it's a MANUAL_PICK // Note: currently, we come here whenever we commit a word. If it's a MANUAL_PICK
// or a DECIDED_WORD we may cancel the commit later; otherwise, we should deactivate // or a DECIDED_WORD we may cancel the commit later; otherwise, we should deactivate
// the last composed word to ensure this does not happen. // the last composed word to ensure this does not happen.
final LastComposedWord lastComposedWord = new LastComposedWord(mEvents, final LastComposedWord lastComposedWord = new LastComposedWord(mEvents,
mInputPointers, mTypedWordCache.toString(), committedWord, separatorString, mInputPointers, mTypedWordCache.toString(), committedWord, separatorString,
prevWord, mCapitalizedMode); prevWordsInfo, mCapitalizedMode);
mInputPointers.reset(); mInputPointers.reset();
if (type != LastComposedWord.COMMIT_TYPE_DECIDED_WORD if (type != LastComposedWord.COMMIT_TYPE_DECIDED_WORD
&& type != LastComposedWord.COMMIT_TYPE_MANUAL_PICK) { && type != LastComposedWord.COMMIT_TYPE_MANUAL_PICK) {

33
java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
View File

@ -37,6 +37,7 @@ import com.android.inputmethod.latin.InputPointers;
import com.android.inputmethod.latin.LastComposedWord; import com.android.inputmethod.latin.LastComposedWord;
import com.android.inputmethod.latin.LatinIME; import com.android.inputmethod.latin.LatinIME;
import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.RichInputConnection; import com.android.inputmethod.latin.RichInputConnection;
import com.android.inputmethod.latin.Suggest; import com.android.inputmethod.latin.Suggest;
import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback;
@ -1233,7 +1234,7 @@ public final class InputLogic {
} }
private void performAdditionToUserHistoryDictionary(final SettingsValues settingsValues, private void performAdditionToUserHistoryDictionary(final SettingsValues settingsValues,
final String suggestion, final String prevWord) { final String suggestion, final PrevWordsInfo prevWordsInfo) {
// If correction is not enabled, we don't add words to the user history dictionary. // If correction is not enabled, we don't add words to the user history dictionary.
// That's to avoid unintended additions in some sensitive fields, or fields that // That's to avoid unintended additions in some sensitive fields, or fields that
// expect to receive non-words. // expect to receive non-words.
@ -1244,8 +1245,8 @@ public final class InputLogic {
mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps(); mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps();
final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds( final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds(
System.currentTimeMillis()); System.currentTimeMillis());
mSuggest.mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized, prevWord, mSuggest.mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized,
timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive); prevWordsInfo, timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive);
} }
public void performUpdateSuggestionStripSync(final SettingsValues settingsValues) { public void performUpdateSuggestionStripSync(final SettingsValues settingsValues) {
@ -1370,13 +1371,15 @@ public final class InputLogic {
} }
} }
final int[] codePoints = StringUtils.toCodePointArray(typedWord); final int[] codePoints = StringUtils.toCodePointArray(typedWord);
mWordComposer.setComposingWord(codePoints,
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints),
getNthPreviousWordForSuggestion(settingsValues.mSpacingAndPunctuations,
// We want the previous word for suggestion. If we have chars in the word // We want the previous word for suggestion. If we have chars in the word
// before the cursor, then we want the word before that, hence 2; otherwise, // before the cursor, then we want the word before that, hence 2; otherwise,
// we want the word immediately before the cursor, hence 1. // we want the word immediately before the cursor, hence 1.
0 == numberOfCharsInWordBeforeCursor ? 1 : 2)); final String prevWord = getNthPreviousWordForSuggestion(
settingsValues.mSpacingAndPunctuations,
0 == numberOfCharsInWordBeforeCursor ? 1 : 2).toString();
mWordComposer.setComposingWord(codePoints,
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints),
new PrevWordsInfo(prevWord));
mWordComposer.setCursorPositionWithinWord( mWordComposer.setCursorPositionWithinWord(
typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor)); typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor));
mConnection.setComposingRegion(expectedCursorPosition - numberOfCharsInWordBeforeCursor, mConnection.setComposingRegion(expectedCursorPosition - numberOfCharsInWordBeforeCursor,
@ -1431,7 +1434,7 @@ public final class InputLogic {
* @param inputTransaction The transaction in progress. * @param inputTransaction The transaction in progress.
*/ */
private void revertCommit(final InputTransaction inputTransaction) { private void revertCommit(final InputTransaction inputTransaction) {
final String previousWord = mLastComposedWord.mPrevWord; final PrevWordsInfo prevWordsInfo = mLastComposedWord.mPrevWordsInfo;
final CharSequence originallyTypedWord = mLastComposedWord.mTypedWord; final CharSequence originallyTypedWord = mLastComposedWord.mTypedWord;
final CharSequence committedWord = mLastComposedWord.mCommittedWord; final CharSequence committedWord = mLastComposedWord.mCommittedWord;
final String committedWordString = committedWord.toString(); final String committedWordString = committedWord.toString();
@ -1453,9 +1456,9 @@ public final class InputLogic {
} }
} }
mConnection.deleteSurroundingText(deleteLength, 0); mConnection.deleteSurroundingText(deleteLength, 0);
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && !TextUtils.isEmpty(committedWord)) {
mSuggest.mDictionaryFacilitator.cancelAddingUserHistory( mSuggest.mDictionaryFacilitator.cancelAddingUserHistory(
previousWord, committedWordString); prevWordsInfo, committedWordString);
} }
final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString; final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
final SpannableString textToCommit = new SpannableString(stringToCommit); final SpannableString textToCommit = new SpannableString(stringToCommit);
@ -1504,7 +1507,7 @@ public final class InputLogic {
// with the typed word, so we need to resume suggestions right away. // with the typed word, so we need to resume suggestions right away.
final int[] codePoints = StringUtils.toCodePointArray(stringToCommit); final int[] codePoints = StringUtils.toCodePointArray(stringToCommit);
mWordComposer.setComposingWord(codePoints, mWordComposer.setComposingWord(codePoints,
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), previousWord); mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), prevWordsInfo);
mConnection.setComposingText(textToCommit, 1); mConnection.setComposingText(textToCommit, 1);
} }
if (inputTransaction.mSettingsValues.mIsInternal) { if (inputTransaction.mSettingsValues.mIsInternal) {
@ -1968,17 +1971,17 @@ public final class InputLogic {
suggestedWords); suggestedWords);
// Use the 2nd previous word as the previous word because the 1st previous word is the word // Use the 2nd previous word as the previous word because the 1st previous word is the word
// to be committed. // to be committed.
final String prevWord = mConnection.getNthPreviousWord( final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(mConnection.getNthPreviousWord(
settingsValues.mSpacingAndPunctuations, 2); settingsValues.mSpacingAndPunctuations, 2));
mConnection.commitText(chosenWordWithSuggestions, 1); mConnection.commitText(chosenWordWithSuggestions, 1);
// Add the word to the user history dictionary // Add the word to the user history dictionary
performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWord); performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWordsInfo);
// TODO: figure out here if this is an auto-correct or if the best word is actually // TODO: figure out here if this is an auto-correct or if the best word is actually
// what user typed. Note: currently this is done much later in // what user typed. Note: currently this is done much later in
// LastComposedWord#didCommitTypedWord by string equality of the remembered // LastComposedWord#didCommitTypedWord by string equality of the remembered
// strings. // strings.
mLastComposedWord = mWordComposer.commitWord(commitType, mLastComposedWord = mWordComposer.commitWord(commitType,
chosenWordWithSuggestions, separatorString, prevWord); chosenWordWithSuggestions, separatorString, prevWordsInfo);
final boolean shouldDiscardPreviousWordForSuggestion; final boolean shouldDiscardPreviousWordForSuggestion;
if (0 == StringUtils.codePointCount(separatorString)) { if (0 == StringUtils.codePointCount(separatorString)) {
// Separator is 0-length, we can keep the previous word for suggestion. Either this // Separator is 0-length, we can keep the previous word for suggestion. Either this

28
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
View File

@ -22,6 +22,7 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import java.io.File; import java.io.File;
import java.util.Locale; import java.util.Locale;
@ -52,29 +53,32 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
} }
/** /**
* Pair will be added to the user history dictionary. * Add a word to the user history dictionary.
* *
* The first word may be null. That means we don't know the context, in other words, * @param userHistoryDictionary the user history dictionary
* it's only a unigram. The first word may also be an empty string : this means start * @param prevWordsInfo the information of previous words
* context, as in beginning of a sentence for example. * @param word the word the user inputted
* The second word may not be null (a NullPointerException would be thrown). * @param isValid whether the word is valid or not
* @param timestamp the timestamp when the word has been inputted
*/ */
public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary,
final String word0, final String word1, final boolean isValid, final int timestamp) { final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid,
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || final int timestamp) {
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { final String prevWord = prevWordsInfo.mPrevWord;
if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return; return;
} }
final int frequency = isValid ? final int frequency = isValid ?
FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
userHistoryDictionary.addWordDynamically(word1, frequency, null /* shortcutTarget */, userHistoryDictionary.addUnigramEntry(word, frequency, null /* shortcutTarget */,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp); 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp);
// Do not insert a word as a bigram of itself // Do not insert a word as a bigram of itself
if (word1.equals(word0)) { if (word.equals(prevWord)) {
return; return;
} }
if (null != word0) { if (null != prevWord) {
userHistoryDictionary.addBigramDynamically(word0, word1, frequency, timestamp); userHistoryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp);
} }
} }
} }

2
java/src/com/android/inputmethod/latin/utils/DistracterFilter.java
View File

@ -114,7 +114,7 @@ public class DistracterFilter {
final int[] codePoints = StringUtils.toCodePointArray(testedWord); final int[] codePoints = StringUtils.toCodePointArray(testedWord);
final int[] coordinates; final int[] coordinates;
coordinates = mKeyboard.getCoordinates(codePoints); coordinates = mKeyboard.getCoordinates(codePoints);
composer.setComposingWord(codePoints, coordinates, prevWordsInfo.mPrevWord); composer.setComposingWord(codePoints, coordinates, prevWordsInfo);
final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
final String consideredWord = trailingSingleQuotesCount > 0 ? final String consideredWord = trailingSingleQuotesCount > 0 ?

47
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -65,7 +65,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
final int probability) { final int probability) {
binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
false /* isNotAWord */, false /* isBlacklisted */, false /* isNotAWord */, false /* isBlacklisted */,
mCurrentTime /* timestamp */); mCurrentTime /* timestamp */);
@ -73,10 +73,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
final String word1, final int probability) { final String word1, final int probability) {
binaryDictionary.addBigramWords(word0, word1, probability, binaryDictionary.addNgramEntry(new PrevWordsInfo(word0), word1, probability,
mCurrentTime /* timestamp */); mCurrentTime /* timestamp */);
} }
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.isValidNgram(new PrevWordsInfo(word0), word1);
}
private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
// 30 days. // 30 days.
final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS); final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS);
@ -224,19 +229,19 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
assertTrue(binaryDictionary.isValidWord("b")); assertTrue(binaryDictionary.isValidWord("b"));
addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b"));
addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b"));
addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "c")); assertTrue(isValidBigram(binaryDictionary, "a", "c"));
// Add bigrams of not valid unigrams. // Add bigrams of not valid unigrams.
addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y"));
addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y"));
binaryDictionary.close(); binaryDictionary.close();
dictFile.delete(); dictFile.delete();
@ -276,9 +281,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary);
assertFalse(binaryDictionary.isValidBigram("a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b"));
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
@ -289,11 +294,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary);
assertTrue(binaryDictionary.isValidBigram("a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingLongTime(binaryDictionary); forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidBigram("a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b"));
binaryDictionary.close(); binaryDictionary.close();
dictFile.delete(); dictFile.delete();
@ -549,8 +554,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
for (int j = 0; j < weakBigramTypedCount; j++) { for (int j = 0; j < weakBigramTypedCount; j++) {
addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY);
} }
assertTrue(binaryDictionary.isValidBigram(strong, target)); assertTrue(isValidBigram(binaryDictionary, strong, target));
assertTrue(binaryDictionary.isValidBigram(weak, target)); assertTrue(isValidBigram(binaryDictionary, weak, target));
for (int i = 0; i < bigramCount; i++) { for (int i = 0; i < bigramCount; i++) {
final int word0Index = random.nextInt(words.size()); final int word0Index = random.nextInt(words.size());
@ -571,8 +576,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
Integer.parseInt(binaryDictionary.getPropertyForTest( Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY)); BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC); assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
assertTrue(binaryDictionary.isValidBigram(strong, target)); assertTrue(isValidBigram(binaryDictionary, strong, target));
assertFalse(binaryDictionary.isValidBigram(weak, target)); assertFalse(isValidBigram(binaryDictionary, weak, target));
break; break;
} }
} }
@ -606,9 +611,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
assertFalse(binaryDictionary.isValidBigram("aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion)); assertTrue(binaryDictionary.migrateTo(toFormatVersion));
@ -619,10 +624,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.isValidWord("bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
binaryDictionary.close(); binaryDictionary.close();
dictFile.delete(); dictFile.delete();
} }

158
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -170,7 +170,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, validLongWord, probability); addUnigramWord(binaryDictionary, validLongWord, probability);
addUnigramWord(binaryDictionary, invalidLongWord, probability); addUnigramWord(binaryDictionary, invalidLongWord, probability);
// Too long short cut. // Too long short cut.
binaryDictionary.addUnigramWord("a", probability, invalidLongWord, binaryDictionary.addUnigramEntry("a", probability, invalidLongWord,
10 /* shortcutProbability */, false /* isNotAWord */, false /* isBlacklisted */, 10 /* shortcutProbability */, false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP); BinaryDictionary.NOT_A_VALID_TIMESTAMP);
addUnigramWord(binaryDictionary, "abc", probability); addUnigramWord(binaryDictionary, "abc", probability);
@ -188,20 +188,35 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile.delete(); dictFile.delete();
} }
private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
final int probability) { final int probability) {
binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
false /* isNotAWord */, false /* isBlacklisted */, false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
} }
private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, private static void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
final String word1, final int probability) { final String word1, final int probability) {
binaryDictionary.addBigramWords(word0, word1, probability, binaryDictionary.addNgramEntry(new PrevWordsInfo(word0), word1, probability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
} }
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.isValidNgram(new PrevWordsInfo(word0), word1);
}
private static void removeBigramEntry(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
binaryDictionary.removeNgramEntry(new PrevWordsInfo(word0), word1);
}
private static int getBigramProbability(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.getNgramProbability(new PrevWordsInfo(word0), word1);
}
public void testAddUnigramWord() { public void testAddUnigramWord() {
for (final int formatVersion : DICT_FORMAT_VERSIONS) { for (final int formatVersion : DICT_FORMAT_VERSIONS) {
testAddUnigramWord(formatVersion); testAddUnigramWord(formatVersion);
@ -312,32 +327,32 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertTrue(binaryDictionary.isValidBigram("aaa", "abb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa")); assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc")); assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
} }
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability, assertEquals(updatedBigramProbability,
binaryDictionary.getBigramProbability("aaa", "abb")); getBigramProbability(binaryDictionary, "aaa", "abb"));
} }
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa")); assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY, assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("bcc", "aaa")); getBigramProbability(binaryDictionary, "bcc", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY, assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("bcc", "bbc")); getBigramProbability(binaryDictionary, "bcc", "bbc"));
assertEquals(Dictionary.NOT_A_PROBABILITY, assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("aaa", "aaa")); getBigramProbability(binaryDictionary, "aaa", "aaa"));
// Testing bigram link. // Testing bigram link.
addUnigramWord(binaryDictionary, "abcde", unigramProbability); addUnigramWord(binaryDictionary, "abcde", unigramProbability);
@ -349,14 +364,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, assertEquals(bigramProbability,
binaryDictionary.getBigramProbability("abcde", "fghij")); getBigramProbability(binaryDictionary, "abcde", "fghij"));
} }
assertEquals(Dictionary.NOT_A_PROBABILITY, assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("abcde", "fgh")); getBigramProbability(binaryDictionary, "abcde", "fgh"));
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability, assertEquals(updatedBigramProbability,
binaryDictionary.getBigramProbability("abcde", "fghij")); getBigramProbability(binaryDictionary, "abcde", "fghij"));
} }
dictFile.delete(); dictFile.delete();
@ -418,10 +433,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigramWords) { for (final Pair<String, String> bigram : bigramWords) {
final int bigramProbability = bigramProbabilities.get(bigram); final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second)); getBigramProbability(binaryDictionary, bigram.first, bigram.second));
} }
} }
@ -454,28 +469,28 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertTrue(binaryDictionary.isValidBigram("aaa", "abb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa")); assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc")); assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb"); removeBigramEntry(binaryDictionary, "aaa", "abb");
assertFalse(binaryDictionary.isValidBigram("aaa", "abb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "abb"));
addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
assertTrue(binaryDictionary.isValidBigram("aaa", "abb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
binaryDictionary.removeBigramWords("aaa", "bcc"); removeBigramEntry(binaryDictionary, "aaa", "bcc");
assertFalse(binaryDictionary.isValidBigram("aaa", "bcc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bcc"));
binaryDictionary.removeBigramWords("abb", "aaa"); removeBigramEntry(binaryDictionary, "abb", "aaa");
assertFalse(binaryDictionary.isValidBigram("abb", "aaa")); assertFalse(isValidBigram(binaryDictionary, "abb", "aaa"));
binaryDictionary.removeBigramWords("abb", "bcc"); removeBigramEntry(binaryDictionary, "abb", "bcc");
assertFalse(binaryDictionary.isValidBigram("abb", "bcc")); assertFalse(isValidBigram(binaryDictionary, "abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb"); removeBigramEntry(binaryDictionary, "aaa", "abb");
// Test remove non-existing bigram operation. // Test remove non-existing bigram operation.
binaryDictionary.removeBigramWords("aaa", "abb"); removeBigramEntry(binaryDictionary, "aaa", "abb");
binaryDictionary.removeBigramWords("bcc", "aaa"); removeBigramEntry(binaryDictionary, "bcc", "aaa");
dictFile.delete(); dictFile.delete();
} }
@ -570,14 +585,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
} }
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa")); assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
binaryDictionary.close(); binaryDictionary.close();
@ -649,10 +664,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigramWords) { for (final Pair<String, String> bigram : bigramWords) {
final int bigramProbability = bigramProbabilities.get(bigram); final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second)); getBigramProbability(binaryDictionary, bigram.first, bigram.second));
} }
} }
@ -742,7 +757,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final Pair<String, String> bigram = bigramWords.get(bigramIndex); final Pair<String, String> bigram = bigramWords.get(bigramIndex);
bigramWords.remove(bigramIndex); bigramWords.remove(bigramIndex);
bigramProbabilities.remove(bigram); bigramProbabilities.remove(bigram);
binaryDictionary.removeBigramWords(bigram.first, bigram.second); removeBigramEntry(binaryDictionary, bigram.first, bigram.second);
} }
} }
@ -765,10 +780,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(probability, assertEquals(probability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second)); getBigramProbability(binaryDictionary, bigram.first, bigram.second));
} }
assertEquals(probability != Dictionary.NOT_A_PROBABILITY, assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
} }
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
binaryDictionary.close(); binaryDictionary.close();
@ -946,10 +961,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final String word1 = entry.getKey().second; final String word1 = entry.getKey().second;
final int bigramProbability = entry.getValue(); final int bigramProbability = entry.getValue();
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(word0, word1)); isValidBigram(binaryDictionary, word0, word1));
if (canCheckBigramProbability(formatVersion)) { if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(word0, word1)); getBigramProbability(binaryDictionary, word0, word1));
} }
} }
} }
@ -993,7 +1008,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final boolean isNotAWord = random.nextBoolean(); final boolean isNotAWord = random.nextBoolean();
final boolean isBlacklisted = random.nextBoolean(); final boolean isBlacklisted = random.nextBoolean();
// TODO: Add tests for historical info. // TODO: Add tests for historical info.
binaryDictionary.addUnigramWord(word, unigramProbability, binaryDictionary.addUnigramEntry(word, unigramProbability,
null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY, null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY,
isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP); isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@ -1023,8 +1038,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = wordProbabilities.get(word1); final int unigramProbability = wordProbabilities.get(word1);
final int bigramProbability = final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability); unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability, addBigramWords(binaryDictionary, word0, word1, bigramProbability);
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
} }
@ -1112,8 +1126,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = wordProbabilitiesToCheckLater.get(word1); final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
final int bigramProbability = final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability); unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability, addBigramWords(binaryDictionary, word0, word1, bigramProbability);
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
} }
@ -1174,7 +1187,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = 100; final int unigramProbability = 100;
final int shortcutProbability = 10; final int shortcutProbability = 10;
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz", binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */); 0 /* timestamp */);
WordProperty wordProperty = binaryDictionary.getWordProperty("aaa"); WordProperty wordProperty = binaryDictionary.getWordProperty("aaa");
@ -1182,7 +1195,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability());
final int updatedShortcutProbability = 2; final int updatedShortcutProbability = 2;
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz", binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */); 0 /* timestamp */);
wordProperty = binaryDictionary.getWordProperty("aaa"); wordProperty = binaryDictionary.getWordProperty("aaa");
@ -1190,7 +1203,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(updatedShortcutProbability, assertEquals(updatedShortcutProbability,
wordProperty.mShortcutTargets.get(0).getProbability()); wordProperty.mShortcutTargets.get(0).getProbability());
binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy", binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */); 0 /* timestamp */);
final HashMap<String, Integer> shortcutTargets = new HashMap<String, Integer>(); final HashMap<String, Integer> shortcutTargets = new HashMap<String, Integer>();
@ -1261,7 +1274,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int shortcutProbability = random.nextInt(0xF); final int shortcutProbability = random.nextInt(0xF);
final String word = words.get(random.nextInt(words.size())); final String word = words.get(random.nextInt(words.size()));
final int unigramProbability = unigramProbabilities.get(word); final int unigramProbability = unigramProbabilities.get(word);
binaryDictionary.addUnigramWord(word, unigramProbability, shortcutTarget, binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget,
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */); 0 /* timestamp */);
if (shortcutTargets.containsKey(word)) { if (shortcutTargets.containsKey(word)) {
@ -1317,14 +1330,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = 150; final int bigramProbability = 150;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability); addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
final int shortcutProbability = 10; final int shortcutProbability = 10;
binaryDictionary.addUnigramWord("ccc", unigramProbability, "xxx", shortcutProbability, binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability,
false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */);
binaryDictionary.addUnigramWord("ddd", unigramProbability, null /* shortcutTarget */, binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY, true /* isNotAWord */, Dictionary.NOT_A_PROBABILITY, true /* isNotAWord */,
true /* isBlacklisted */, 0 /* timestamp */); true /* isBlacklisted */, 0 /* timestamp */);
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion)); assertTrue(binaryDictionary.migrateTo(toFormatVersion));
assertTrue(binaryDictionary.isValidDictionary()); assertTrue(binaryDictionary.isValidDictionary());
@ -1332,9 +1345,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
if (canCheckBigramProbability(toFormatVersion)) { if (canCheckBigramProbability(toFormatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bbb")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb"));
} }
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc"); WordProperty wordProperty = binaryDictionary.getWordProperty("ccc");
assertEquals(1, wordProperty.mShortcutTargets.size()); assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord);
@ -1395,8 +1408,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = unigramProbabilities.get(word1); final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability = final int bigramProbability =
random.nextInt(0xFF - unigramProbability) + unigramProbability; random.nextInt(0xFF - unigramProbability) + unigramProbability;
binaryDictionary.addBigramWords(word0, word1, bigramProbability, addBigramWords(binaryDictionary, word0, word1, bigramProbability);
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC();
} }
@ -1415,9 +1427,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigrams) { for (final Pair<String, String> bigram : bigrams) {
if (canCheckBigramProbability(toFormatVersion)) { if (canCheckBigramProbability(toFormatVersion)) {
assertEquals((int)bigramProbabilities.get(bigram), assertEquals((int)bigramProbabilities.get(bigram),
binaryDictionary.getBigramProbability(bigram.first, bigram.second)); getBigramProbability(binaryDictionary, bigram.first, bigram.second));
} }
assertTrue(binaryDictionary.isValidBigram(bigram.first, bigram.second)); assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
} }
assertEquals(bigramProbabilities.size(), Integer.parseInt( assertEquals(bigramProbabilities.size(), Integer.parseInt(
binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY)));

39
tests/src/com/android/inputmethod/latin/WordComposerTests.java
View File

@ -40,8 +40,8 @@ public class WordComposerTests extends AndroidTestCase {
final int[] COORDINATES_WITHIN_BMP = final int[] COORDINATES_WITHIN_BMP =
CoordinateUtils.newCoordinateArray(CODEPOINTS_WITHIN_BMP.length, CoordinateUtils.newCoordinateArray(CODEPOINTS_WITHIN_BMP.length,
Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
final String PREVWORD = "prevword"; final PrevWordsInfo PREV_WORDS_INFO = new PrevWordsInfo("prevword");
wc.setComposingWord(CODEPOINTS_WITHIN_BMP, COORDINATES_WITHIN_BMP, PREVWORD); wc.setComposingWord(CODEPOINTS_WITHIN_BMP, COORDINATES_WITHIN_BMP, PREV_WORDS_INFO);
assertEquals(wc.size(), STR_WITHIN_BMP.codePointCount(0, STR_WITHIN_BMP.length())); assertEquals(wc.size(), STR_WITHIN_BMP.codePointCount(0, STR_WITHIN_BMP.length()));
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
wc.setCursorPositionWithinWord(2); wc.setCursorPositionWithinWord(2);
@ -57,7 +57,7 @@ public class WordComposerTests extends AndroidTestCase {
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1));
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
// Check the previous word is still there // Check the previous word is still there
assertEquals(PREVWORD, wc.getPrevWordsInfoForSuggestion().mPrevWord); assertEquals(PREV_WORDS_INFO, wc.getPrevWordsInfoForSuggestion());
// Move the cursor past the end of the word // Move the cursor past the end of the word
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1));
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15));
@ -74,7 +74,7 @@ public class WordComposerTests extends AndroidTestCase {
CoordinateUtils.newCoordinateArray(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length, CoordinateUtils.newCoordinateArray(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length,
Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
null /* previousWord */); new PrevWordsInfo(null));
assertEquals(wc.size(), CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length); assertEquals(wc.size(), CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length);
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
wc.setCursorPositionWithinWord(3); wc.setCursorPositionWithinWord(3);
@ -85,46 +85,53 @@ public class WordComposerTests extends AndroidTestCase {
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord); assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord);
final PrevWordsInfo PREV_WORDS_INFO_STR_WITHIN_BMP = new PrevWordsInfo(STR_WITHIN_BMP);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
STR_WITHIN_BMP); PREV_WORDS_INFO_STR_WITHIN_BMP);
wc.setCursorPositionWithinWord(3); wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7));
assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord); assertEquals(PREV_WORDS_INFO_STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion());
final PrevWordsInfo PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR =
new PrevWordsInfo(STR_WITH_SUPPLEMENTARY_CHAR);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
STR_WITH_SUPPLEMENTARY_CHAR); PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR);
wc.setCursorPositionWithinWord(3); wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7));
assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord); assertEquals(PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR,
wc.getPrevWordsInfoForSuggestion());
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
STR_WITHIN_BMP); PREV_WORDS_INFO_STR_WITHIN_BMP);
wc.setCursorPositionWithinWord(3); wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3));
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1));
assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord); assertEquals(PREV_WORDS_INFO_STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion());
final PrevWordsInfo PREV_WORDS_INFO_NULL = new PrevWordsInfo(null);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
null /* previousWord */); PREV_WORDS_INFO_NULL);
wc.setCursorPositionWithinWord(3); wc.setCursorPositionWithinWord(3);
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9));
assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord); assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
STR_WITH_SUPPLEMENTARY_CHAR); PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10));
assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord); assertEquals(PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR,
wc.getPrevWordsInfoForSuggestion());
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
null /* previousWord */); PREV_WORDS_INFO_NULL);
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-11)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-11));
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
null /* previousWord */); PREV_WORDS_INFO_NULL);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0));
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
null /* previousWord */); PREV_WORDS_INFO_NULL);
wc.setCursorPositionWithinWord(2); wc.setCursorPositionWithinWord(2);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0)); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0));
} }

9
tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -19,6 +19,7 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
@ -74,13 +75,13 @@ public class Ver4DictEncoder implements DictEncoder {
for (final WordProperty wordProperty : dict) { for (final WordProperty wordProperty : dict) {
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts // TODO: switch to addMultipleDictionaryEntries when they support shortcuts
if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) { if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(), binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
null /* shortcutTarget */, 0 /* shortcutProbability */, null /* shortcutTarget */, 0 /* shortcutProbability */,
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry, wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */); 0 /* timestamp */);
} else { } else {
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(), binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
shortcutTarget.mWord, shortcutTarget.getProbability(), shortcutTarget.mWord, shortcutTarget.getProbability(),
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry, wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */); 0 /* timestamp */);
@ -93,8 +94,8 @@ public class Ver4DictEncoder implements DictEncoder {
for (final WordProperty word0Property : dict) { for (final WordProperty word0Property : dict) {
if (null == word0Property.mBigrams) continue; if (null == word0Property.mBigrams) continue;
for (final WeightedString word1 : word0Property.mBigrams) { for (final WeightedString word1 : word0Property.mBigrams) {
binaryDict.addBigramWords(word0Property.mWord, word1.mWord, word1.getProbability(), binaryDict.addNgramEntry(new PrevWordsInfo(word0Property.mWord), word1.mWord,
0 /* timestamp */); word1.getProbability(), 0 /* timestamp */);
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC(); binaryDict.flushWithGC();
} }

13
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -21,6 +21,7 @@ import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log; import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.FileUtils;
@ -109,11 +110,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
} }
private static void addToDict(final UserHistoryDictionary dict, final List<String> words) { private static void addToDict(final UserHistoryDictionary dict, final List<String> words) {
String prevWord = null; PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
for (String word : words) { for (String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWord, word, true, UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true,
(int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())); (int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()));
prevWord = word; prevWordsInfo = new PrevWordsInfo(word);
} }
} }
@ -260,10 +261,10 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final UserHistoryDictionary dict = final UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(), dummyLocale); PersonalizationHelper.getUserHistoryDictionary(getContext(), dummyLocale);
dict.waitAllTasksForTests(); dict.waitAllTasksForTests();
String prevWord = null; PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
for (final String word : words) { for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWord, word, true, mCurrentTime); UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime);
prevWord = word; prevWordsInfo = new PrevWordsInfo(word);
dict.waitAllTasksForTests(); dict.waitAllTasksForTests();
assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word)); assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word));
} }