Merge "Apply distracter filter for UserHistoryDictionary."

main
Keisuke Kuroyanagi 2014-05-26 16:28:22 +00:00 committed by Android (Google) Code Review
commit 71c795d00b
6 changed files with 45 additions and 20 deletions

View File

@ -168,7 +168,7 @@ public class DictionaryFacilitator {
} }
public DictionaryFacilitator() { public DictionaryFacilitator() {
mDistracterFilter = new DistracterFilter.EmptyDistracterFilter(); mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
} }
public DictionaryFacilitator(final DistracterFilter distracterFilter) { public DictionaryFacilitator(final DistracterFilter distracterFilter) {
@ -448,7 +448,7 @@ public class DictionaryFacilitator {
// We don't add words with 0-frequency (assuming they would be profanity etc.). // We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0; final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord, UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord,
isValid, timeStampInSeconds); isValid, timeStampInSeconds, mDistracterFilter);
} }
public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo, public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo,

View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.CombinedFormatUtils; import com.android.inputmethod.latin.utils.CombinedFormatUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.LanguageModelParam;
@ -271,9 +272,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** /**
* Adds unigram information of a word to the dictionary. May overwrite an existing entry. * Adds unigram information of a word to the dictionary. May overwrite an existing entry.
*/ */
public void addUnigramEntry(final String word, final int frequency, public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) { final boolean isBlacklisted, final int timestamp,
final DistracterFilter distracterFilter) {
reloadDictionaryIfRequired(); reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() { asyncExecuteTaskWithWriteLock(new Runnable() {
@Override @Override
@ -281,6 +283,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (mBinaryDictionary == null) { if (mBinaryDictionary == null) {
return; return;
} }
if (distracterFilter.isDistracterToWordsInDictionaries(
PrevWordsInfo.EMPTY_PREV_WORDS_INFO, word, mLocale)) {
// The word is a distracter.
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */); runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp); isNotAWord, isBlacklisted, timestamp);

View File

@ -23,6 +23,7 @@ import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.DistracterFilter;
import java.io.File; import java.io.File;
import java.util.Locale; import java.util.Locale;
@ -60,10 +61,11 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
* @param word the word the user inputted * @param word the word the user inputted
* @param isValid whether the word is valid or not * @param isValid whether the word is valid or not
* @param timestamp the timestamp when the word has been inputted * @param timestamp the timestamp when the word has been inputted
* @param distracterFilter the filter to check whether the word is a distracter
*/ */
public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary,
final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid, final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid,
final int timestamp) { final int timestamp, final DistracterFilter distracterFilter) {
final String prevWord = prevWordsInfo.mPrevWord; final String prevWord = prevWordsInfo.mPrevWord;
if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { (prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
@ -71,8 +73,9 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
} }
final int frequency = isValid ? final int frequency = isValid ?
FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
userHistoryDictionary.addUnigramEntry(word, frequency, null /* shortcutTarget */, userHistoryDictionary.addUnigramEntryWithCheckingDistracter(word, frequency,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp); null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, timestamp, distracterFilter);
// Do not insert a word as a bigram of itself // Do not insert a word as a bigram of itself
if (word.equals(prevWord)) { if (word.equals(prevWord)) {
return; return;

View File

@ -40,7 +40,7 @@ public interface DistracterFilter {
public void close(); public void close();
public static final class EmptyDistracterFilter implements DistracterFilter { public static final DistracterFilter EMPTY_DISTRACTER_FILTER = new DistracterFilter() {
@Override @Override
public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo, public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo,
String testedWord, Locale locale) { String testedWord, Locale locale) {
@ -54,5 +54,5 @@ public interface DistracterFilter {
@Override @Override
public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) { public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) {
} }
} };
} }

View File

@ -32,6 +32,7 @@ import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId; import com.android.inputmethod.keyboard.KeyboardId;
import com.android.inputmethod.keyboard.KeyboardLayoutSet; import com.android.inputmethod.keyboard.KeyboardLayoutSet;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
@ -205,14 +206,25 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
final String consideredWord = trailingSingleQuotesCount > 0 ? final String consideredWord = trailingSingleQuotesCount > 0 ?
testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
testedWord; testedWord;
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, mKeyboard.getProximityInfo(), @Override
true /* blockOffensiveWords */, null /* additionalFeaturesOptions */, public void run() {
0 /* sessionId */, null /* rawSuggestions */); final SuggestionResults suggestionResults =
mDictionaryFacilitator.getSuggestionResults(
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
null /* additionalFeaturesOptions */, 0 /* sessionId */,
null /* rawSuggestions */);
if (suggestionResults.isEmpty()) { if (suggestionResults.isEmpty()) {
return false; holder.set(false);
return;
} }
return isDistracter(suggestionResults, consideredWord); holder.set(isDistracter(suggestionResults, consideredWord));
}
});
// It's OK to block the distracter filtering, but the dictionary lookup should be done
// sequentially using ExecutorUtils.
return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
} }
} }

View File

@ -23,6 +23,7 @@ import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.FileUtils;
import java.io.File; import java.io.File;
@ -112,7 +113,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO;
for (String word : words) { for (String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true,
(int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())); (int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
DistracterFilter.EMPTY_DISTRACTER_FILTER);
prevWordsInfo = new PrevWordsInfo(word); prevWordsInfo = new PrevWordsInfo(word);
} }
} }
@ -262,7 +264,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
dict.waitAllTasksForTests(); dict.waitAllTasksForTests();
PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null); PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
for (final String word : words) { for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime); UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
prevWordsInfo = new PrevWordsInfo(word); prevWordsInfo = new PrevWordsInfo(word);
dict.waitAllTasksForTests(); dict.waitAllTasksForTests();
assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word)); assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word));