From 166d8c2293eee268c62ecfba65d2d89d35d97540 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Fri, 6 Jun 2014 19:38:44 +0900 Subject: [PATCH] Make Distracter filter use getMaxFrequencyOfExactMatches(). Bug: 13142176 Bug: 15428247 Change-Id: I5c23fbea2851f891f76f19d9da2cb70ae964569b --- .../latin/DictionaryCollection.java | 14 +- .../DistracterFilterUsingSuggestion.java | 155 ++---------------- .../latin/DistracterFilterTest.java | 65 ++++++-- 3 files changed, 77 insertions(+), 157 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java index 53be28139..53c78fd00 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java +++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java @@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary { int maxFreq = -1; for (int i = mDictionaries.size() - 1; i >= 0; --i) { final int tempFreq = mDictionaries.get(i).getFrequency(word); - if (tempFreq >= maxFreq) { - maxFreq = tempFreq; - } + maxFreq = Math.max(tempFreq, maxFreq); + } + return maxFreq; + } + + @Override + public int getMaxFrequencyOfExactMatches(final String word) { + int maxFreq = -1; + for (int i = mDictionaries.size() - 1; i >= 0; --i) { + final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word); + maxFreq = Math.max(tempFreq, maxFreq); } return maxFreq; } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java index b9c7f5671..8c3844ed8 100644 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java @@ -16,33 +16,22 @@ package com.android.inputmethod.latin.utils; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.TimeUnit; import android.content.Context; -import android.content.res.Resources; -import android.text.InputType; import android.util.Log; -import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputMethodSubtype; -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardId; -import com.android.inputmethod.keyboard.KeyboardLayoutSet; -import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.PrevWordsInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; /** * This class is used to prevent distracters being added to personalization * or user history dictionaries */ +// TODO: Rename. public class DistracterFilterUsingSuggestion implements DistracterFilter { private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName(); private static final boolean DEBUG = false; @@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter { private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; private final Context mContext; - private final Map mLocaleToSubtypeMap; - private final Map mLocaleToKeyboardMap; private final DictionaryFacilitator mDictionaryFacilitator; - private Keyboard mKeyboard; private final Object mLock = new Object(); /** @@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter { */ public DistracterFilterUsingSuggestion(final Context context) { mContext = context; - mLocaleToSubtypeMap = new HashMap<>(); - mLocaleToKeyboardMap = new HashMap<>(); mDictionaryFacilitator = new DictionaryFacilitator(); - mKeyboard = null; } @Override @@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter { @Override public void updateEnabledSubtypes(final List enabledSubtypes) { - final Map newLocaleToSubtypeMap = new HashMap<>(); - if (enabledSubtypes != null) { - for (final InputMethodSubtype subtype : enabledSubtypes) { - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - if (newLocaleToSubtypeMap.containsKey(locale)) { - // Multiple subtypes are enabled for one locale. - // TODO: Investigate what we should do for this case. - continue; - } - newLocaleToSubtypeMap.put(locale, subtype); - } - } - if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) { - // Enabled subtypes have not been changed. - return; - } - synchronized (mLock) { - mLocaleToSubtypeMap.clear(); - mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap); - mLocaleToKeyboardMap.clear(); - } - } - - private boolean isDistracter( - final SuggestionResults suggestionResults, final String consideredWord) { - int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY; - for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) { - if (suggestedWordInfo.mWord.equals(consideredWord)) { - perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord); - continue; - } - // Exact match can include case errors, accent errors, digraph conversions. - final boolean isExactMatch = suggestedWordInfo.isExactMatch(); - final boolean isExactMatchWithIntentionalOmission = - suggestedWordInfo.isExactMatchWithIntentionalOmission(); - - if (DEBUG) { - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore); - Log.d(TAG, "consideredWord: " + consideredWord); - Log.d(TAG, "top suggestion: " + suggestedWordInfo.mWord); - Log.d(TAG, "suggestionScore: " + suggestedWordInfo.mScore); - Log.d(TAG, "normalizedScore: " + normalizedScore); - Log.d(TAG, "isExactMatch: " + isExactMatch); - Log.d(TAG, "isExactMatchWithIntentionalOmission: " - + isExactMatchWithIntentionalOmission); - } - if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) { - final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency( - suggestedWordInfo.mWord); - if (DEBUG) { - Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability); - Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability); - } - if (perfectMatchProbability > topNonPerfectProbability) { - return false; - } - } - return isExactMatch || isExactMatchWithIntentionalOmission; - } - return false; - } - - private void loadKeyboardForLocale(final Locale newLocale) { - final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); - if (cachedKeyboard != null) { - mKeyboard = cachedKeyboard; - return; - } - final InputMethodSubtype subtype; - synchronized (mLock) { - subtype = mLocaleToSubtypeMap.get(newLocale); - } - if (subtype == null) { - return; - } - final EditorInfo editorInfo = new EditorInfo(); - editorInfo.inputType = InputType.TYPE_CLASS_TEXT; - final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( - mContext, editorInfo); - final Resources res = mContext.getResources(); - final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); - final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); - builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); - builder.setSubtype(subtype); - builder.setIsSpellChecker(false /* isSpellChecker */); - final KeyboardLayoutSet layoutSet = builder.build(); - mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); } private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { @@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter { } if (!locale.equals(mDictionaryFacilitator.getLocale())) { synchronized (mLock) { - if (!mLocaleToSubtypeMap.containsKey(locale)) { - Log.e(TAG, "Locale " + locale + " is not enabled."); - // TODO: Investigate what we should do for disabled locales. - return false; - } - loadKeyboardForLocale(locale); // Reset dictionaries for the locale. try { loadDictionariesForLocale(locale); @@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter { } } } - if (mKeyboard == null) { - return false; + // The tested word is a distracter when there is a word that is exact matched to the tested + // word and its probability is higher than the tested word's probability. + final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord); + final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord); + final boolean isDistracter = perfectMatchFreq < exactMatchFreq; + if (DEBUG) { + Log.d(TAG, "testedWord: " + testedWord); + Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq); + Log.d(TAG, "exactMatchFreq: " + exactMatchFreq); + Log.d(TAG, "isDistracter: " + isDistracter); } - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates = mKeyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO); - - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final AsyncResultHolder holder = new AsyncResultHolder<>(); - ExecutorUtils.getExecutor("check distracters").execute(new Runnable() { - @Override - public void run() { - final SuggestionResults suggestionResults = - mDictionaryFacilitator.getSuggestionResults( - composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, - mKeyboard.getProximityInfo(), true /* blockOffensiveWords */, - null /* additionalFeaturesOptions */, 0 /* sessionId */, - null /* rawSuggestions */); - if (suggestionResults.isEmpty()) { - holder.set(false); - return; - } - holder.set(isDistracter(suggestionResults, consideredWord)); - } - }); - // It's OK to block the distracter filtering, but the dictionary lookup should be done - // sequentially using ExecutorUtils. - return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + return isDistracter; } } diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java index 406e9a9b8..b7f2271be 100644 --- a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java +++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java @@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase { final Locale localeEnUs = new Locale("en", "US"); String typedWord; - typedWord = "google"; - // For this test case, we consider "google" is a distracter to "Google". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - typedWord = "Bill"; // For this test case, we consider "Bill" is a distracter to "bill". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( @@ -83,13 +78,18 @@ public class DistracterFilterTest extends InputTestsBase { EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "cafe"; - // For this test case, we consider "café" is not a distracter to any word in dictionaries. + // For this test case, we consider "cafe" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + typedWord = "I'll"; + // For this test case, we consider "I'll" is not a distracter to any word in dictionaries. assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "ill"; - // For this test case, we consider "ill" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + // For this test case, we consider "ill" is a distracter to "I'll" + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "asdfd"; @@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase { typedWord = "thank"; // For this test case, we consider "thank" is not a distracter to any other word // in dictionaries. - assertFalse( - mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); + + final Locale localeDeDe = new Locale("de", "DE"); + + typedWord = "fuer"; + // For this test case, we consider "fuer" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + typedWord = "fUEr"; + // For this test case, we consider "fUEr" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + typedWord = "fur"; + // For this test case, we consider "fur" is a distracter to "für". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); + + final Locale localeFrFr = new Locale("fr", "FR"); + + typedWord = "a"; + // For this test case, we consider "a" is a distracter to "à". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "à"; + // For this test case, we consider "à" is not a distracter to any word in dictionaries. + assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "etre"; + // For this test case, we consider "etre" is a distracter to "être". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "États-unis"; + // For this test case, we consider "États-unis" is a distracter to "États-Unis". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); + + typedWord = "ÉtatsUnis"; + // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis". + assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( + EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); } }