nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Revert "Use PrevWordsInfo for get/add/remove n-gram(bigram) entry.""

main
Ken Wakasa 2014-05-20 14:01:33 +00:00 committed by Android (Google) Code Review
parent 305778b53a 4e21d3711f
commit 06dd0ef877
15 changed files with 209 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -359,16 +359,14 @@ public final class BinaryDictionary extends Dictionary {
}
@UsedForTesting
public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) {
return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY;
public boolean isValidBigram(final String word0, final String word1) {
return getBigramProbability(word0, word1) != NOT_A_PROBABILITY;
}
public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) {
if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
return NOT_A_PROBABILITY;
}
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word);
public int getBigramProbability(final String word0, final String word1) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return NOT_A_PROBABILITY;
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
@ -419,7 +417,7 @@ public final class BinaryDictionary extends Dictionary {
}
// Add a unigram entry to binary dictionary with unigram attributes in native code.
public void addUnigramEntry(final String word, final int probability,
public void addUnigramWord(final String word, final int probability,
final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
if (TextUtils.isEmpty(word)) {
@ -433,26 +431,25 @@ public final class BinaryDictionary extends Dictionary {
mHasUpdated = true;
}
// Add an n-gram entry to the binary dictionary with timestamp in native code.
public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
final int probability,
// Add a bigram entry to binary dictionary with timestamp in native code.
public void addBigramWords(final String word0, final String word1, final int probability,
final int timestamp) {
if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word);
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp);
mHasUpdated = true;
}
// Remove an n-gram entry from the binary dictionary in native code.
public void removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) {
if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) {
// Remove a bigram entry form binary dictionary in native code.
public void removeBigramWords(final String word0, final String word1) {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
final int[] codePoints1 = StringUtils.toCodePointArray(word);
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
mHasUpdated = true;
}

16
java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
View File

@ -142,7 +142,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
Log.d(TAG, "loadAccountVocabulary: " + word);
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */,
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
}
@ -224,7 +224,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
*/
private void addNameLocked(final String name) {
int len = StringUtils.codePointCount(name);
PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
String prevWord = null;
// TODO: Better tokenization for non-Latin writing systems
for (int i = 0; i < len; i++) {
if (Character.isLetter(name.codePointAt(i))) {
@ -239,19 +239,19 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
final int wordLen = StringUtils.codePointCount(word);
if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
if (DEBUG) {
Log.d(TAG, "addName " + name + ", " + word + ", "
+ prevWordsInfo.mPrevWord);
Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, FREQUENCY_FOR_CONTACTS,
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS,
null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && mUseFirstLastBigrams) {
if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addNgramEntryLocked(prevWordsInfo, word, FREQUENCY_FOR_CONTACTS_BIGRAM,
addBigramDynamicallyLocked(prevWord, word,
FREQUENCY_FOR_CONTACTS_BIGRAM,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
}
prevWordsInfo = new PrevWordsInfo(word);
prevWord = word;
}
}
}

20
java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java
View File

@ -370,23 +370,22 @@ public class DictionaryFacilitatorForSuggest {
}
public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized,
final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds,
final String previousWord, final int timeStampInSeconds,
final boolean blockPotentiallyOffensive) {
final Dictionaries dictionaries = mDictionaries;
final String[] words = suggestion.split(Constants.WORD_SEPARATOR);
for (int i = 0; i < words.length; i++) {
final String currentWord = words[i];
final PrevWordsInfo prevWordsInfoForCurrentWord =
(i == 0) ? prevWordsInfo : new PrevWordsInfo(words[i - 1]);
final String prevWord = (i == 0) ? previousWord : words[i - 1];
final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false;
addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord,
addWordToUserHistory(dictionaries, prevWord, currentWord,
wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive);
}
}
private void addWordToUserHistory(final Dictionaries dictionaries,
final PrevWordsInfo prevWordsInfo, final String word, final boolean wasAutoCapitalized,
final int timeStampInSeconds, final boolean blockPotentiallyOffensive) {
private void addWordToUserHistory(final Dictionaries dictionaries, final String prevWord,
final String word, final boolean wasAutoCapitalized, final int timeStampInSeconds,
final boolean blockPotentiallyOffensive) {
final ExpandableBinaryDictionary userHistoryDictionary =
dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary == null) {
@ -431,16 +430,15 @@ public class DictionaryFacilitatorForSuggest {
// We demote unrecognized words (frequency < 0, below) by specifying them as "invalid".
// We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord,
UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWord, secondWord,
isValid, timeStampInSeconds);
}
public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo,
final String committedWord) {
public void cancelAddingUserHistory(final String previousWord, final String committedWord) {
final ExpandableBinaryDictionary userHistoryDictionary =
mDictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary != null) {
userHistoryDictionary.removeNgramDynamically(prevWordsInfo, committedWord);
userHistoryDictionary.removeBigramDynamically(previousWord, committedWord);
}
}

30
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -269,9 +269,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
/**
* Adds unigram information of a word to the dictionary. May overwrite an existing entry.
* Dynamically adds a word unigram to the dictionary. May overwrite an existing entry.
*/
public void addUnigramEntry(final String word, final int frequency,
public void addWordDynamically(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
reloadDictionaryIfRequired();
@ -282,23 +282,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp);
}
});
}
protected void addUnigramLocked(final String word, final int frequency,
protected void addWordDynamicallyLocked(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq,
mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp);
}
/**
* Adds n-gram information of a word to the dictionary. May overwrite an existing entry.
* Dynamically adds a word bigram in the dictionary. May overwrite an existing entry.
*/
public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
public void addBigramDynamically(final String word0, final String word1,
final int frequency, final int timestamp) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
@ -308,20 +308,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addNgramEntryLocked(prevWordsInfo, word, frequency, timestamp);
addBigramDynamicallyLocked(word0, word1, frequency, timestamp);
}
});
}
protected void addNgramEntryLocked(final PrevWordsInfo prevWordsInfo, final String word,
protected void addBigramDynamicallyLocked(final String word0, final String word1,
final int frequency, final int timestamp) {
mBinaryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp);
mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp);
}
/**
* Dynamically remove the n-gram entry in the dictionary.
* Dynamically remove a word bigram in the dictionary.
*/
public void removeNgramDynamically(final PrevWordsInfo prevWordsInfo, final String word1) {
public void removeBigramDynamically(final String word0, final String word1) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
@Override
@ -330,7 +330,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
mBinaryDictionary.removeNgramEntry(prevWordsInfo, word1);
mBinaryDictionary.removeBigramWords(word0, word1);
}
});
}
@ -428,9 +428,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return mBinaryDictionary.isValidWord(word);
}
protected boolean isValidNgramLocked(final PrevWordsInfo prevWordsInfo, final String word) {
protected boolean isValidBigramLocked(final String word1, final String word2) {
if (mBinaryDictionary == null) return false;
return mBinaryDictionary.isValidNgram(prevWordsInfo, word);
return mBinaryDictionary.isValidBigram(word1, word2);
}
/**

6
java/src/com/android/inputmethod/latin/LastComposedWord.java
View File

@ -48,7 +48,7 @@ public final class LastComposedWord {
public final String mTypedWord;
public final CharSequence mCommittedWord;
public final String mSeparatorString;
public final PrevWordsInfo mPrevWordsInfo;
public final String mPrevWord;
public final int mCapitalizedMode;
public final InputPointers mInputPointers =
new InputPointers(Constants.DICTIONARY_MAX_WORD_LENGTH);
@ -64,7 +64,7 @@ public final class LastComposedWord {
public LastComposedWord(final ArrayList<Event> events,
final InputPointers inputPointers, final String typedWord,
final CharSequence committedWord, final String separatorString,
final PrevWordsInfo prevWordsInfo, final int capitalizedMode) {
final String prevWord, final int capitalizedMode) {
if (inputPointers != null) {
mInputPointers.copy(inputPointers);
}
@ -73,7 +73,7 @@ public final class LastComposedWord {
mCommittedWord = committedWord;
mSeparatorString = separatorString;
mActive = true;
mPrevWordsInfo = prevWordsInfo;
mPrevWord = prevWord;
mCapitalizedMode = capitalizedMode;
}

4
java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
View File

@ -258,12 +258,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
// Safeguard against adding really long words.
if (word.length() < MAX_WORD_LENGTH) {
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, adjustedFrequency, null /* shortcutTarget */,
addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */,
0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(shortcut, adjustedFrequency, word,
addWordDynamicallyLocked(shortcut, adjustedFrequency, word,
USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */,
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
}

11
java/src/com/android/inputmethod/latin/WordComposer.java
View File

@ -294,10 +294,11 @@ public final class WordComposer {
* This will register NOT_A_COORDINATE for X and Ys, and use the passed keyboard for proximity.
* @param codePoints the code points to set as the composing word.
* @param coordinates the x, y coordinates of the key in the CoordinateUtils format
* @param prevWordsInfo the information of previous words, to use as context for suggestions
* @param previousWord the previous word, to use as context for suggestions. Can be null if
* the context is nil (typically, at start of text).
*/
public void setComposingWord(final int[] codePoints, final int[] coordinates,
final PrevWordsInfo prevWordsInfo) {
final CharSequence previousWord) {
reset();
final int length = codePoints.length;
for (int i = 0; i < length; ++i) {
@ -306,7 +307,7 @@ public final class WordComposer {
CoordinateUtils.yFromArray(coordinates, i)));
}
mIsResumed = true;
mPrevWordsInfo = prevWordsInfo;
mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString());
}
/**
@ -412,13 +413,13 @@ public final class WordComposer {
// `type' should be one of the LastComposedWord.COMMIT_TYPE_* constants above.
// committedWord should contain suggestion spans if applicable.
public LastComposedWord commitWord(final int type, final CharSequence committedWord,
final String separatorString, final PrevWordsInfo prevWordsInfo) {
final String separatorString, final String prevWord) {
// Note: currently, we come here whenever we commit a word. If it's a MANUAL_PICK
// or a DECIDED_WORD we may cancel the commit later; otherwise, we should deactivate
// the last composed word to ensure this does not happen.
final LastComposedWord lastComposedWord = new LastComposedWord(mEvents,
mInputPointers, mTypedWordCache.toString(), committedWord, separatorString,
prevWordsInfo, mCapitalizedMode);
prevWord, mCapitalizedMode);
mInputPointers.reset();
if (type != LastComposedWord.COMMIT_TYPE_DECIDED_WORD
&& type != LastComposedWord.COMMIT_TYPE_MANUAL_PICK) {

35
java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java
View File

@ -37,7 +37,6 @@ import com.android.inputmethod.latin.InputPointers;
import com.android.inputmethod.latin.LastComposedWord;
import com.android.inputmethod.latin.LatinIME;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.RichInputConnection;
import com.android.inputmethod.latin.Suggest;
import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback;
@ -1234,7 +1233,7 @@ public final class InputLogic {
}
private void performAdditionToUserHistoryDictionary(final SettingsValues settingsValues,
final String suggestion, final PrevWordsInfo prevWordsInfo) {
final String suggestion, final String prevWord) {
// If correction is not enabled, we don't add words to the user history dictionary.
// That's to avoid unintended additions in some sensitive fields, or fields that
// expect to receive non-words.
@ -1245,8 +1244,8 @@ public final class InputLogic {
mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps();
final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds(
System.currentTimeMillis());
mSuggest.mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized,
prevWordsInfo, timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive);
mSuggest.mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized, prevWord,
timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive);
}
public void performUpdateSuggestionStripSync(final SettingsValues settingsValues) {
@ -1371,15 +1370,13 @@ public final class InputLogic {
}
}
final int[] codePoints = StringUtils.toCodePointArray(typedWord);
// We want the previous word for suggestion. If we have chars in the word
// before the cursor, then we want the word before that, hence 2; otherwise,
// we want the word immediately before the cursor, hence 1.
final String prevWord = getNthPreviousWordForSuggestion(
settingsValues.mSpacingAndPunctuations,
0 == numberOfCharsInWordBeforeCursor ? 1 : 2).toString();
mWordComposer.setComposingWord(codePoints,
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints),
new PrevWordsInfo(prevWord));
getNthPreviousWordForSuggestion(settingsValues.mSpacingAndPunctuations,
// We want the previous word for suggestion. If we have chars in the word
// before the cursor, then we want the word before that, hence 2; otherwise,
// we want the word immediately before the cursor, hence 1.
0 == numberOfCharsInWordBeforeCursor ? 1 : 2));
mWordComposer.setCursorPositionWithinWord(
typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor));
mConnection.setComposingRegion(expectedCursorPosition - numberOfCharsInWordBeforeCursor,
@ -1434,7 +1431,7 @@ public final class InputLogic {
* @param inputTransaction The transaction in progress.
*/
private void revertCommit(final InputTransaction inputTransaction) {
final PrevWordsInfo prevWordsInfo = mLastComposedWord.mPrevWordsInfo;
final String previousWord = mLastComposedWord.mPrevWord;
final CharSequence originallyTypedWord = mLastComposedWord.mTypedWord;
final CharSequence committedWord = mLastComposedWord.mCommittedWord;
final String committedWordString = committedWord.toString();
@ -1456,9 +1453,9 @@ public final class InputLogic {
}
}
mConnection.deleteSurroundingText(deleteLength, 0);
if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && !TextUtils.isEmpty(committedWord)) {
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
mSuggest.mDictionaryFacilitator.cancelAddingUserHistory(
prevWordsInfo, committedWordString);
previousWord, committedWordString);
}
final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
final SpannableString textToCommit = new SpannableString(stringToCommit);
@ -1507,7 +1504,7 @@ public final class InputLogic {
// with the typed word, so we need to resume suggestions right away.
final int[] codePoints = StringUtils.toCodePointArray(stringToCommit);
mWordComposer.setComposingWord(codePoints,
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), prevWordsInfo);
mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), previousWord);
mConnection.setComposingText(textToCommit, 1);
}
if (inputTransaction.mSettingsValues.mIsInternal) {
@ -1971,17 +1968,17 @@ public final class InputLogic {
suggestedWords);
// Use the 2nd previous word as the previous word because the 1st previous word is the word
// to be committed.
final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(mConnection.getNthPreviousWord(
settingsValues.mSpacingAndPunctuations, 2));
final String prevWord = mConnection.getNthPreviousWord(
settingsValues.mSpacingAndPunctuations, 2);
mConnection.commitText(chosenWordWithSuggestions, 1);
// Add the word to the user history dictionary
performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWordsInfo);
performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWord);
// TODO: figure out here if this is an auto-correct or if the best word is actually
// what user typed. Note: currently this is done much later in
// LastComposedWord#didCommitTypedWord by string equality of the remembered
// strings.
mLastComposedWord = mWordComposer.commitWord(commitType,
chosenWordWithSuggestions, separatorString, prevWordsInfo);
chosenWordWithSuggestions, separatorString, prevWord);
final boolean shouldDiscardPreviousWordForSuggestion;
if (0 == StringUtils.codePointCount(separatorString)) {
// Separator is 0-length, we can keep the previous word for suggestion. Either this

28
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
View File

@ -22,7 +22,6 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import java.io.File;
import java.util.Locale;
@ -53,32 +52,29 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
}
/**
* Add a word to the user history dictionary.
* Pair will be added to the user history dictionary.
*
* @param userHistoryDictionary the user history dictionary
* @param prevWordsInfo the information of previous words
* @param word the word the user inputted
* @param isValid whether the word is valid or not
* @param timestamp the timestamp when the word has been inputted
* The first word may be null. That means we don't know the context, in other words,
* it's only a unigram. The first word may also be an empty string : this means start
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary,
final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid,
final int timestamp) {
final String prevWord = prevWordsInfo.mPrevWord;
if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
final String word0, final String word1, final boolean isValid, final int timestamp) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
}
final int frequency = isValid ?
FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
userHistoryDictionary.addUnigramEntry(word, frequency, null /* shortcutTarget */,
userHistoryDictionary.addWordDynamically(word1, frequency, null /* shortcutTarget */,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp);
// Do not insert a word as a bigram of itself
if (word.equals(prevWord)) {
if (word1.equals(word0)) {
return;
}
if (null != prevWord) {
userHistoryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp);
if (null != word0) {
userHistoryDictionary.addBigramDynamically(word0, word1, frequency, timestamp);
}
}
}

2
java/src/com/android/inputmethod/latin/utils/DistracterFilter.java
View File

@ -114,7 +114,7 @@ public class DistracterFilter {
final int[] codePoints = StringUtils.toCodePointArray(testedWord);
final int[] coordinates;
coordinates = mKeyboard.getCoordinates(codePoints);
composer.setComposingWord(codePoints, coordinates, prevWordsInfo);
composer.setComposingWord(codePoints, coordinates, prevWordsInfo.mPrevWord);
final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
final String consideredWord = trailingSingleQuotesCount > 0 ?

47
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -65,7 +65,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
final int probability) {
binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */,
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
false /* isNotAWord */, false /* isBlacklisted */,
mCurrentTime /* timestamp */);
@ -73,15 +73,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
final String word1, final int probability) {
binaryDictionary.addNgramEntry(new PrevWordsInfo(word0), word1, probability,
binaryDictionary.addBigramWords(word0, word1, probability,
mCurrentTime /* timestamp */);
}
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.isValidNgram(new PrevWordsInfo(word0), word1);
}
private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
// 30 days.
final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS);
@ -229,19 +224,19 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
assertTrue(binaryDictionary.isValidWord("b"));
addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
assertFalse(binaryDictionary.isValidBigram("a", "b"));
addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
assertTrue(binaryDictionary.isValidBigram("a", "b"));
addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "a", "c"));
assertTrue(binaryDictionary.isValidBigram("a", "c"));
// Add bigrams of not valid unigrams.
addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY);
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
assertFalse(binaryDictionary.isValidBigram("x", "y"));
addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY);
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
assertFalse(binaryDictionary.isValidBigram("x", "y"));
binaryDictionary.close();
dictFile.delete();
@ -281,9 +276,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
assertTrue(binaryDictionary.isValidBigram("a", "b"));
forcePassingShortTime(binaryDictionary);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
assertFalse(binaryDictionary.isValidBigram("a", "b"));
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
@ -294,11 +289,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
assertTrue(binaryDictionary.isValidBigram("a", "b"));
forcePassingShortTime(binaryDictionary);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
assertTrue(binaryDictionary.isValidBigram("a", "b"));
forcePassingLongTime(binaryDictionary);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
assertFalse(binaryDictionary.isValidBigram("a", "b"));
binaryDictionary.close();
dictFile.delete();
@ -554,8 +549,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
for (int j = 0; j < weakBigramTypedCount; j++) {
addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY);
}
assertTrue(isValidBigram(binaryDictionary, strong, target));
assertTrue(isValidBigram(binaryDictionary, weak, target));
assertTrue(binaryDictionary.isValidBigram(strong, target));
assertTrue(binaryDictionary.isValidBigram(weak, target));
for (int i = 0; i < bigramCount; i++) {
final int word0Index = random.nextInt(words.size());
@ -576,8 +571,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
Integer.parseInt(binaryDictionary.getPropertyForTest(
BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
assertTrue(isValidBigram(binaryDictionary, strong, target));
assertFalse(isValidBigram(binaryDictionary, weak, target));
assertTrue(binaryDictionary.isValidBigram(strong, target));
assertFalse(binaryDictionary.isValidBigram(weak, target));
break;
}
}
@ -611,9 +606,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY);
addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abc"));
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertFalse(binaryDictionary.isValidBigram("aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion));
@ -624,10 +619,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
assertTrue(binaryDictionary.isValidWord("bbb"));
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "bbb"));
addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
binaryDictionary.close();
dictFile.delete();
}

158
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -170,7 +170,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, validLongWord, probability);
addUnigramWord(binaryDictionary, invalidLongWord, probability);
// Too long short cut.
binaryDictionary.addUnigramEntry("a", probability, invalidLongWord,
binaryDictionary.addUnigramWord("a", probability, invalidLongWord,
10 /* shortcutProbability */, false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
addUnigramWord(binaryDictionary, "abc", probability);
@ -188,35 +188,20 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile.delete();
}
private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
final int probability) {
binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */,
BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
false /* isNotAWord */, false /* isBlacklisted */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
}
private static void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
final String word1, final int probability) {
binaryDictionary.addNgramEntry(new PrevWordsInfo(word0), word1, probability,
binaryDictionary.addBigramWords(word0, word1, probability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
}
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.isValidNgram(new PrevWordsInfo(word0), word1);
}
private static void removeBigramEntry(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
binaryDictionary.removeNgramEntry(new PrevWordsInfo(word0), word1);
}
private static int getBigramProbability(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
return binaryDictionary.getNgramProbability(new PrevWordsInfo(word0), word1);
}
public void testAddUnigramWord() {
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
testAddUnigramWord(formatVersion);
@ -327,32 +312,32 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
}
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability,
getBigramProbability(binaryDictionary, "aaa", "abb"));
binaryDictionary.getBigramProbability("aaa", "abb"));
}
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
getBigramProbability(binaryDictionary, "bcc", "aaa"));
binaryDictionary.getBigramProbability("bcc", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
getBigramProbability(binaryDictionary, "bcc", "bbc"));
binaryDictionary.getBigramProbability("bcc", "bbc"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
getBigramProbability(binaryDictionary, "aaa", "aaa"));
binaryDictionary.getBigramProbability("aaa", "aaa"));
// Testing bigram link.
addUnigramWord(binaryDictionary, "abcde", unigramProbability);
@ -364,14 +349,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
getBigramProbability(binaryDictionary, "abcde", "fghij"));
binaryDictionary.getBigramProbability("abcde", "fghij"));
}
assertEquals(Dictionary.NOT_A_PROBABILITY,
getBigramProbability(binaryDictionary, "abcde", "fgh"));
binaryDictionary.getBigramProbability("abcde", "fgh"));
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability,
getBigramProbability(binaryDictionary, "abcde", "fghij"));
binaryDictionary.getBigramProbability("abcde", "fghij"));
}
dictFile.delete();
@ -433,10 +418,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigramWords) {
final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.isValidBigram(bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
}
@ -469,28 +454,28 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
removeBigramEntry(binaryDictionary, "aaa", "abb");
assertFalse(isValidBigram(binaryDictionary, "aaa", "abb"));
binaryDictionary.removeBigramWords("aaa", "abb");
assertFalse(binaryDictionary.isValidBigram("aaa", "abb"));
addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
assertTrue(isValidBigram(binaryDictionary, "aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
removeBigramEntry(binaryDictionary, "aaa", "bcc");
assertFalse(isValidBigram(binaryDictionary, "aaa", "bcc"));
removeBigramEntry(binaryDictionary, "abb", "aaa");
assertFalse(isValidBigram(binaryDictionary, "abb", "aaa"));
removeBigramEntry(binaryDictionary, "abb", "bcc");
assertFalse(isValidBigram(binaryDictionary, "abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "bcc");
assertFalse(binaryDictionary.isValidBigram("aaa", "bcc"));
binaryDictionary.removeBigramWords("abb", "aaa");
assertFalse(binaryDictionary.isValidBigram("abb", "aaa"));
binaryDictionary.removeBigramWords("abb", "bcc");
assertFalse(binaryDictionary.isValidBigram("abb", "bcc"));
removeBigramEntry(binaryDictionary, "aaa", "abb");
binaryDictionary.removeBigramWords("aaa", "abb");
// Test remove non-existing bigram operation.
removeBigramEntry(binaryDictionary, "aaa", "abb");
removeBigramEntry(binaryDictionary, "bcc", "aaa");
binaryDictionary.removeBigramWords("aaa", "abb");
binaryDictionary.removeBigramWords("bcc", "aaa");
dictFile.delete();
}
@ -585,14 +570,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
}
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
binaryDictionary.flushWithGC();
binaryDictionary.close();
@ -664,10 +649,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigramWords) {
final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.isValidBigram(bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
}
@ -757,7 +742,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final Pair<String, String> bigram = bigramWords.get(bigramIndex);
bigramWords.remove(bigramIndex);
bigramProbabilities.remove(bigram);
removeBigramEntry(binaryDictionary, bigram.first, bigram.second);
binaryDictionary.removeBigramWords(bigram.first, bigram.second);
}
}
@ -780,10 +765,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (canCheckBigramProbability(formatVersion)) {
assertEquals(probability,
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
binaryDictionary.flushWithGC();
binaryDictionary.close();
@ -961,10 +946,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final String word1 = entry.getKey().second;
final int bigramProbability = entry.getValue();
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, word0, word1));
binaryDictionary.isValidBigram(word0, word1));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
getBigramProbability(binaryDictionary, word0, word1));
binaryDictionary.getBigramProbability(word0, word1));
}
}
}
@ -1008,7 +993,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final boolean isNotAWord = random.nextBoolean();
final boolean isBlacklisted = random.nextBoolean();
// TODO: Add tests for historical info.
binaryDictionary.addUnigramEntry(word, unigramProbability,
binaryDictionary.addUnigramWord(word, unigramProbability,
null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY,
isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@ -1038,7 +1023,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = wordProbabilities.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC();
}
@ -1126,7 +1112,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC();
}
@ -1187,7 +1174,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = 100;
final int shortcutProbability = 10;
binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
WordProperty wordProperty = binaryDictionary.getWordProperty("aaa");
@ -1195,7 +1182,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability());
final int updatedShortcutProbability = 2;
binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
wordProperty = binaryDictionary.getWordProperty("aaa");
@ -1203,7 +1190,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(updatedShortcutProbability,
wordProperty.mShortcutTargets.get(0).getProbability());
binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy",
binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
final HashMap<String, Integer> shortcutTargets = new HashMap<String, Integer>();
@ -1274,7 +1261,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int shortcutProbability = random.nextInt(0xF);
final String word = words.get(random.nextInt(words.size()));
final int unigramProbability = unigramProbabilities.get(word);
binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget,
binaryDictionary.addUnigramWord(word, unigramProbability, shortcutTarget,
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
if (shortcutTargets.containsKey(word)) {
@ -1330,14 +1317,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = 150;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
final int shortcutProbability = 10;
binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability,
binaryDictionary.addUnigramWord("ccc", unigramProbability, "xxx", shortcutProbability,
false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */);
binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */,
binaryDictionary.addUnigramWord("ddd", unigramProbability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY, true /* isNotAWord */,
true /* isBlacklisted */, 0 /* timestamp */);
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion));
assertTrue(binaryDictionary.isValidDictionary());
@ -1345,9 +1332,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
if (canCheckBigramProbability(toFormatVersion)) {
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bbb"));
}
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc");
assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord);
@ -1408,7 +1395,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability =
random.nextInt(0xFF - unigramProbability) + unigramProbability;
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDictionary.flushWithGC();
}
@ -1427,9 +1415,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final Pair<String, String> bigram : bigrams) {
if (canCheckBigramProbability(toFormatVersion)) {
assertEquals((int)bigramProbabilities.get(bigram),
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
assertTrue(binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
assertEquals(bigramProbabilities.size(), Integer.parseInt(
binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY)));

39
tests/src/com/android/inputmethod/latin/WordComposerTests.java
View File

@ -40,8 +40,8 @@ public class WordComposerTests extends AndroidTestCase {
final int[] COORDINATES_WITHIN_BMP =
CoordinateUtils.newCoordinateArray(CODEPOINTS_WITHIN_BMP.length,
Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
final PrevWordsInfo PREV_WORDS_INFO = new PrevWordsInfo("prevword");
wc.setComposingWord(CODEPOINTS_WITHIN_BMP, COORDINATES_WITHIN_BMP, PREV_WORDS_INFO);
final String PREVWORD = "prevword";
wc.setComposingWord(CODEPOINTS_WITHIN_BMP, COORDINATES_WITHIN_BMP, PREVWORD);
assertEquals(wc.size(), STR_WITHIN_BMP.codePointCount(0, STR_WITHIN_BMP.length()));
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
wc.setCursorPositionWithinWord(2);
@ -57,7 +57,7 @@ public class WordComposerTests extends AndroidTestCase {
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1));
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
// Check the previous word is still there
assertEquals(PREV_WORDS_INFO, wc.getPrevWordsInfoForSuggestion());
assertEquals(PREVWORD, wc.getPrevWordsInfoForSuggestion().mPrevWord);
// Move the cursor past the end of the word
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1));
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15));
@ -74,7 +74,7 @@ public class WordComposerTests extends AndroidTestCase {
CoordinateUtils.newCoordinateArray(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length,
Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
new PrevWordsInfo(null));
null /* previousWord */);
assertEquals(wc.size(), CODEPOINTS_WITH_SUPPLEMENTARY_CHAR.length);
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
wc.setCursorPositionWithinWord(3);
@ -85,53 +85,46 @@ public class WordComposerTests extends AndroidTestCase {
assertFalse(wc.isCursorFrontOrMiddleOfComposingWord());
assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord);
final PrevWordsInfo PREV_WORDS_INFO_STR_WITHIN_BMP = new PrevWordsInfo(STR_WITHIN_BMP);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_STR_WITHIN_BMP);
STR_WITHIN_BMP);
wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7));
assertEquals(PREV_WORDS_INFO_STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion());
assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord);
final PrevWordsInfo PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR =
new PrevWordsInfo(STR_WITH_SUPPLEMENTARY_CHAR);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR);
STR_WITH_SUPPLEMENTARY_CHAR);
wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7));
assertEquals(PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR,
wc.getPrevWordsInfoForSuggestion());
assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_STR_WITHIN_BMP);
STR_WITHIN_BMP);
wc.setCursorPositionWithinWord(3);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3));
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1));
assertEquals(PREV_WORDS_INFO_STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion());
assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord);
final PrevWordsInfo PREV_WORDS_INFO_NULL = new PrevWordsInfo(null);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_NULL);
null /* previousWord */);
wc.setCursorPositionWithinWord(3);
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9));
assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR);
STR_WITH_SUPPLEMENTARY_CHAR);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10));
assertEquals(PREV_WORDS_INFO_STR_WITH_SUPPLEMENTARY_CHAR,
wc.getPrevWordsInfoForSuggestion());
assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord);
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_NULL);
null /* previousWord */);
assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-11));
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_NULL);
null /* previousWord */);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0));
wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR,
PREV_WORDS_INFO_NULL);
null /* previousWord */);
wc.setCursorPositionWithinWord(2);
assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(0));
}

9
tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -19,7 +19,6 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
@ -75,13 +74,13 @@ public class Ver4DictEncoder implements DictEncoder {
for (final WordProperty wordProperty : dict) {
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts
if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(),
null /* shortcutTarget */, 0 /* shortcutProbability */,
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */);
} else {
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(),
shortcutTarget.mWord, shortcutTarget.getProbability(),
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */);
@ -94,8 +93,8 @@ public class Ver4DictEncoder implements DictEncoder {
for (final WordProperty word0Property : dict) {
if (null == word0Property.mBigrams) continue;
for (final WeightedString word1 : word0Property.mBigrams) {
binaryDict.addNgramEntry(new PrevWordsInfo(word0Property.mWord), word1.mWord,
word1.getProbability(), 0 /* timestamp */);
binaryDict.addBigramWords(word0Property.mWord, word1.mWord, word1.getProbability(),
0 /* timestamp */);
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();
}

13
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -21,7 +21,6 @@ import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
@ -110,11 +109,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
private static void addToDict(final UserHistoryDictionary dict, final List<String> words) {
PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
String prevWord = null;
for (String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true,
UserHistoryDictionary.addToDictionary(dict, prevWord, word, true,
(int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()));
prevWordsInfo = new PrevWordsInfo(word);
prevWord = word;
}
}
@ -261,10 +260,10 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(), dummyLocale);
dict.waitAllTasksForTests();
PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
String prevWord = null;
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime);
prevWordsInfo = new PrevWordsInfo(word);
UserHistoryDictionary.addToDictionary(dict, prevWord, word, true, mCurrentTime);
prevWord = word;
dict.waitAllTasksForTests();
assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word));
}