nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Make Distracter filter use getMaxFrequencyOfExactMatches()."

main
Keisuke Kuroyanagi 2014-06-06 10:44:03 +00:00 committed by Android (Google) Code Review
parent 8cae9f50b4 166d8c2293
commit 7640bb1505
3 changed files with 77 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
java/src/com/android/inputmethod/latin/DictionaryCollection.java
View File

@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary {
int maxFreq = -1;
for (int i = mDictionaries.size() - 1; i >= 0; --i) {
final int tempFreq = mDictionaries.get(i).getFrequency(word);
if (tempFreq >= maxFreq) {
maxFreq = tempFreq;
}
maxFreq = Math.max(tempFreq, maxFreq);
}
return maxFreq;
}
@Override
public int getMaxFrequencyOfExactMatches(final String word) {
int maxFreq = -1;
for (int i = mDictionaries.size() - 1; i >= 0; --i) {
final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word);
maxFreq = Math.max(tempFreq, maxFreq);
}
return maxFreq;
}

155
java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
View File

@ -16,33 +16,22 @@
package com.android.inputmethod.latin.utils;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import android.content.Context;
import android.content.res.Resources;
import android.text.InputType;
import android.util.Log;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId;
import com.android.inputmethod.keyboard.KeyboardLayoutSet;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
/**
* This class is used to prevent distracters being added to personalization
* or user history dictionaries
*/
// TODO: Rename.
public class DistracterFilterUsingSuggestion implements DistracterFilter {
private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName();
private static final boolean DEBUG = false;
@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;
private final Context mContext;
private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
private final DictionaryFacilitator mDictionaryFacilitator;
private Keyboard mKeyboard;
private final Object mLock = new Object();
/**
@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
*/
public DistracterFilterUsingSuggestion(final Context context) {
mContext = context;
mLocaleToSubtypeMap = new HashMap<>();
mLocaleToKeyboardMap = new HashMap<>();
mDictionaryFacilitator = new DictionaryFacilitator();
mKeyboard = null;
}
@Override
@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
@Override
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>();
if (enabledSubtypes != null) {
for (final InputMethodSubtype subtype : enabledSubtypes) {
final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
if (newLocaleToSubtypeMap.containsKey(locale)) {
// Multiple subtypes are enabled for one locale.
// TODO: Investigate what we should do for this case.
continue;
}
newLocaleToSubtypeMap.put(locale, subtype);
}
}
if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) {
// Enabled subtypes have not been changed.
return;
}
synchronized (mLock) {
mLocaleToSubtypeMap.clear();
mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap);
mLocaleToKeyboardMap.clear();
}
}
private boolean isDistracter(
final SuggestionResults suggestionResults, final String consideredWord) {
int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY;
for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) {
if (suggestedWordInfo.mWord.equals(consideredWord)) {
perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord);
continue;
}
// Exact match can include case errors, accent errors, digraph conversions.
final boolean isExactMatch = suggestedWordInfo.isExactMatch();
final boolean isExactMatchWithIntentionalOmission =
suggestedWordInfo.isExactMatchWithIntentionalOmission();
if (DEBUG) {
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore);
Log.d(TAG, "consideredWord: " + consideredWord);
Log.d(TAG, "top suggestion: " + suggestedWordInfo.mWord);
Log.d(TAG, "suggestionScore: " + suggestedWordInfo.mScore);
Log.d(TAG, "normalizedScore: " + normalizedScore);
Log.d(TAG, "isExactMatch: " + isExactMatch);
Log.d(TAG, "isExactMatchWithIntentionalOmission: "
+ isExactMatchWithIntentionalOmission);
}
if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) {
final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency(
suggestedWordInfo.mWord);
if (DEBUG) {
Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability);
Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability);
}
if (perfectMatchProbability > topNonPerfectProbability) {
return false;
}
}
return isExactMatch || isExactMatchWithIntentionalOmission;
}
return false;
}
private void loadKeyboardForLocale(final Locale newLocale) {
final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale);
if (cachedKeyboard != null) {
mKeyboard = cachedKeyboard;
return;
}
final InputMethodSubtype subtype;
synchronized (mLock) {
subtype = mLocaleToSubtypeMap.get(newLocale);
}
if (subtype == null) {
return;
}
final EditorInfo editorInfo = new EditorInfo();
editorInfo.inputType = InputType.TYPE_CLASS_TEXT;
final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder(
mContext, editorInfo);
final Resources res = mContext.getResources();
final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res);
final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res);
builder.setKeyboardGeometry(keyboardWidth, keyboardHeight);
builder.setSubtype(subtype);
builder.setIsSpellChecker(false /* isSpellChecker */);
final KeyboardLayoutSet layoutSet = builder.build();
mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
}
private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
}
if (!locale.equals(mDictionaryFacilitator.getLocale())) {
synchronized (mLock) {
if (!mLocaleToSubtypeMap.containsKey(locale)) {
Log.e(TAG, "Locale " + locale + " is not enabled.");
// TODO: Investigate what we should do for disabled locales.
return false;
}
loadKeyboardForLocale(locale);
// Reset dictionaries for the locale.
try {
loadDictionariesForLocale(locale);
@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
}
}
}
if (mKeyboard == null) {
return false;
// The tested word is a distracter when there is a word that is exact matched to the tested
// word and its probability is higher than the tested word's probability.
final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);
final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
if (DEBUG) {
Log.d(TAG, "testedWord: " + testedWord);
Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
Log.d(TAG, "exactMatchFreq: " + exactMatchFreq);
Log.d(TAG, "isDistracter: " + isDistracter);
}
final WordComposer composer = new WordComposer();
final int[] codePoints = StringUtils.toCodePointArray(testedWord);
final int[] coordinates = mKeyboard.getCoordinates(codePoints);
composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO);
final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
final String consideredWord = trailingSingleQuotesCount > 0 ?
testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
testedWord;
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
@Override
public void run() {
final SuggestionResults suggestionResults =
mDictionaryFacilitator.getSuggestionResults(
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
null /* additionalFeaturesOptions */, 0 /* sessionId */,
null /* rawSuggestions */);
if (suggestionResults.isEmpty()) {
holder.set(false);
return;
}
holder.set(isDistracter(suggestionResults, consideredWord));
}
});
// It's OK to block the distracter filtering, but the dictionary lookup should be done
// sequentially using ExecutorUtils.
return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
return isDistracter;
}
}

65
tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
View File

@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase {
final Locale localeEnUs = new Locale("en", "US");
String typedWord;
typedWord = "google";
// For this test case, we consider "google" is a distracter to "Google".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "Bill";
// For this test case, we consider "Bill" is a distracter to "bill".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
@ -83,13 +78,18 @@ public class DistracterFilterTest extends InputTestsBase {
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "cafe";
// For this test case, we consider "café" is not a distracter to any word in dictionaries.
// For this test case, we consider "cafe" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "I'll";
// For this test case, we consider "I'll" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "ill";
// For this test case, we consider "ill" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
// For this test case, we consider "ill" is a distracter to "I'll"
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
typedWord = "asdfd";
@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase {
typedWord = "thank";
// For this test case, we consider "thank" is not a distracter to any other word
// in dictionaries.
assertFalse(
mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
final Locale localeDeDe = new Locale("de", "DE");
typedWord = "fuer";
// For this test case, we consider "fuer" is a distracter to "für".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
typedWord = "fUEr";
// For this test case, we consider "fUEr" is a distracter to "für".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
typedWord = "fur";
// For this test case, we consider "fur" is a distracter to "für".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
final Locale localeFrFr = new Locale("fr", "FR");
typedWord = "a";
// For this test case, we consider "a" is a distracter to "à".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
typedWord = "à";
// For this test case, we consider "à" is not a distracter to any word in dictionaries.
assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
typedWord = "etre";
// For this test case, we consider "etre" is a distracter to "être".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
typedWord = "États-unis";
// For this test case, we consider "États-unis" is a distracter to "États-Unis".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
typedWord = "ÉtatsUnis";
// For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis".
assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
}
}