From f498e53933f64a344c7f9321a49d874a57611169 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Fri, 23 May 2014 18:12:37 +0900 Subject: [PATCH] Make DistracterFilter interface. Bug: 13142176 Bug: 15094186 Change-Id: If94b0a155b5ea2ff6b839e7da9d12a9cc6553931 --- .../latin/DictionaryFacilitator.java | 2 +- .../android/inputmethod/latin/LatinIME.java | 4 +- .../latin/utils/DistracterFilter.java | 213 ++-------------- .../DistracterFilterUsingSuggestion.java | 227 ++++++++++++++++++ .../latin/DistracterFilterTest.java | 6 +- 5 files changed, 250 insertions(+), 202 deletions(-) create mode 100644 java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 0f410d574..212363895 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -169,7 +169,7 @@ public class DictionaryFacilitator { } public DictionaryFacilitator() { - mDistracterFilter = new DistracterFilter(); + mDistracterFilter = new DistracterFilter.EmptyDistracterFilter(); } public DictionaryFacilitator(final DistracterFilter distracterFilter) { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 5fc40585a..34d5f714c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -84,7 +84,7 @@ import com.android.inputmethod.latin.utils.CapsModeUtils; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CoordinateUtils; import com.android.inputmethod.latin.utils.DialogUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.utils.DistracterFilterUsingSuggestion; import com.android.inputmethod.latin.utils.ImportantNoticeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; @@ -125,7 +125,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private final Settings mSettings; private final DictionaryFacilitator mDictionaryFacilitator = - new DictionaryFacilitator(new DistracterFilter(this /* context */)); + new DictionaryFacilitator(new DistracterFilterUsingSuggestion(this /* context */)); private final InputLogic mInputLogic = new InputLogic(this /* LatinIME */, this /* SuggestionStripViewAccessor */, mDictionaryFacilitator); // We expect to have only one decoder in almost all cases, hence the default capacity of 1. diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java index 331bc505c..6e0fab32a 100644 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java @@ -16,159 +16,14 @@ package com.android.inputmethod.latin.utils; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import android.content.Context; -import android.content.res.Resources; -import android.text.InputType; -import android.util.Log; -import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputMethodSubtype; -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardId; -import com.android.inputmethod.keyboard.KeyboardLayoutSet; -import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.PrevWordsInfo; -import com.android.inputmethod.latin.Suggest; -import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; -import com.android.inputmethod.latin.SuggestedWords; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; - -/** - * This class is used to prevent distracters being added to personalization - * or user history dictionaries - */ -public class DistracterFilter { - private static final String TAG = DistracterFilter.class.getSimpleName(); - - private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; - - private final Context mContext; - private final Map mLocaleToSubtypeMap; - private final Map mLocaleToKeyboardMap; - private final DictionaryFacilitator mDictionaryFacilitator; - private final Suggest mSuggest; - private Keyboard mKeyboard; - private final Object mLock = new Object(); - - // If the score of the top suggestion exceeds this value, the tested word (e.g., - // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to - // words in dictionary. The greater the threshold is, the less likely the tested word would - // become a distracter, which means the tested word will be more likely to be added to - // the dictionary. - private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; - - // Create empty distracter filter. - public DistracterFilter() { - mContext = null; - mLocaleToSubtypeMap = new HashMap<>(); - mLocaleToKeyboardMap = new HashMap<>(); - // TODO: Quit assigning null. - mDictionaryFacilitator = null; - mSuggest = null; - mKeyboard = null; - } - - /** - * Create a DistracterFilter instance. - * - * @param context the context. - */ - public DistracterFilter(final Context context) { - mContext = context; - mLocaleToSubtypeMap = new HashMap<>(); - mLocaleToKeyboardMap = new HashMap<>(); - mDictionaryFacilitator = new DictionaryFacilitator(); - mSuggest = new Suggest(mDictionaryFacilitator); - mKeyboard = null; - } - - public void close() { - if (mDictionaryFacilitator != null) { - mDictionaryFacilitator.closeDictionaries(); - } - } - - public void updateEnabledSubtypes(final List enabledSubtypes) { - final Map newLocaleToSubtypeMap = new HashMap<>(); - if (enabledSubtypes != null) { - for (final InputMethodSubtype subtype : enabledSubtypes) { - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - if (newLocaleToSubtypeMap.containsKey(locale)) { - // Multiple subtypes are enabled for one locale. - // TODO: Investigate what we should do for this case. - continue; - } - newLocaleToSubtypeMap.put(locale, subtype); - } - } - if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) { - // Enabled subtypes have not been changed. - return; - } - synchronized (mLock) { - mLocaleToSubtypeMap.clear(); - mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap); - mLocaleToKeyboardMap.clear(); - } - } - - private static boolean suggestionExceedsDistracterThreshold( - final SuggestedWordInfo suggestion, final String consideredWord, - final float distracterThreshold) { - if (null != suggestion) { - final int suggestionScore = suggestion.mScore; - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestion.mWord, suggestionScore); - if (normalizedScore > distracterThreshold) { - return true; - } - } - return false; - } - - private void loadKeyboardForLocale(final Locale newLocale) { - final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); - if (cachedKeyboard != null) { - mKeyboard = cachedKeyboard; - return; - } - final InputMethodSubtype subtype; - synchronized (mLock) { - subtype = mLocaleToSubtypeMap.get(newLocale); - } - if (subtype == null) { - return; - } - final EditorInfo editorInfo = new EditorInfo(); - editorInfo.inputType = InputType.TYPE_CLASS_TEXT; - final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( - mContext, editorInfo); - final Resources res = mContext.getResources(); - final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); - final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); - builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); - builder.setSubtype(subtype); - builder.setIsSpellChecker(false /* isSpellChecker */); - final KeyboardLayoutSet layoutSet = builder.build(); - mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); - } - - private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { - mDictionaryFacilitator.resetDictionaries(mContext, newlocale, - false /* useContactsDict */, false /* usePersonalizedDicts */, - false /* forceReloadMainDictionary */, null /* listener */); - mDictionaryFacilitator.waitForLoadingMainDictionary( - TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); - } +public interface DistracterFilter { /** * Determine whether a word is a distracter to words in dictionaries. * @@ -179,59 +34,25 @@ public class DistracterFilter { * @return true if testedWord is a distracter, otherwise false. */ public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, - final String testedWord, final Locale locale) { - if (mSuggest == null || locale == null) { + final String testedWord, final Locale locale); + + public void updateEnabledSubtypes(final List enabledSubtypes); + + public void close(); + + public static final class EmptyDistracterFilter implements DistracterFilter { + @Override + public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo, + String testedWord, Locale locale) { return false; } - if (!locale.equals(mDictionaryFacilitator.getLocale())) { - synchronized (mLock) { - if (!mLocaleToSubtypeMap.containsKey(locale)) { - Log.e(TAG, "Locale " + locale + " is not enabled."); - // TODO: Investigate what we should do for disabled locales. - return false; - } - loadKeyboardForLocale(locale); - // Reset dictionaries for the locale. - try { - loadDictionariesForLocale(locale); - } catch (final InterruptedException e) { - Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", - e); - return false; - } - } - } - if (mKeyboard == null) { - return false; - } - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates = mKeyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates, prevWordsInfo); - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final AsyncResultHolder holder = new AsyncResultHolder(); - final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { - @Override - public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - if (suggestedWords != null && suggestedWords.size() > 1) { - // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from - // the decoder is at index 1. - final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); - final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( - firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); - holder.set(hasStrongDistractor); - } - } - }; - mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), - true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, - null /* additionalFeaturesOptions */, 0 /* sessionId */, - SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + @Override + public void close() { + } - return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + @Override + public void updateEnabledSubtypes(List enabledSubtypes) { + } } } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java new file mode 100644 index 000000000..92033b76f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import android.content.Context; +import android.content.res.Resources; +import android.text.InputType; +import android.util.Log; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.keyboard.KeyboardId; +import com.android.inputmethod.keyboard.KeyboardLayoutSet; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.DictionaryFacilitator; +import com.android.inputmethod.latin.PrevWordsInfo; +import com.android.inputmethod.latin.Suggest; +import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; +import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.WordComposer; + +/** + * This class is used to prevent distracters being added to personalization + * or user history dictionaries + */ +public class DistracterFilterUsingSuggestion implements DistracterFilter { + private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName(); + + private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; + + private final Context mContext; + private final Map mLocaleToSubtypeMap; + private final Map mLocaleToKeyboardMap; + private final DictionaryFacilitator mDictionaryFacilitator; + private final Suggest mSuggest; + private Keyboard mKeyboard; + private final Object mLock = new Object(); + + // If the score of the top suggestion exceeds this value, the tested word (e.g., + // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to + // words in dictionary. The greater the threshold is, the less likely the tested word would + // become a distracter, which means the tested word will be more likely to be added to + // the dictionary. + private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; + + /** + * Create a DistracterFilter instance. + * + * @param context the context. + */ + public DistracterFilterUsingSuggestion(final Context context) { + mContext = context; + mLocaleToSubtypeMap = new HashMap<>(); + mLocaleToKeyboardMap = new HashMap<>(); + mDictionaryFacilitator = new DictionaryFacilitator(); + mSuggest = new Suggest(mDictionaryFacilitator); + mKeyboard = null; + } + + @Override + public void close() { + mDictionaryFacilitator.closeDictionaries(); + } + + @Override + public void updateEnabledSubtypes(final List enabledSubtypes) { + final Map newLocaleToSubtypeMap = new HashMap<>(); + if (enabledSubtypes != null) { + for (final InputMethodSubtype subtype : enabledSubtypes) { + final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); + if (newLocaleToSubtypeMap.containsKey(locale)) { + // Multiple subtypes are enabled for one locale. + // TODO: Investigate what we should do for this case. + continue; + } + newLocaleToSubtypeMap.put(locale, subtype); + } + } + if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) { + // Enabled subtypes have not been changed. + return; + } + synchronized (mLock) { + mLocaleToSubtypeMap.clear(); + mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap); + mLocaleToKeyboardMap.clear(); + } + } + + private static boolean suggestionExceedsDistracterThreshold( + final SuggestedWordInfo suggestion, final String consideredWord, + final float distracterThreshold) { + if (null != suggestion) { + final int suggestionScore = suggestion.mScore; + final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( + consideredWord, suggestion.mWord, suggestionScore); + if (normalizedScore > distracterThreshold) { + return true; + } + } + return false; + } + + private void loadKeyboardForLocale(final Locale newLocale) { + final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); + if (cachedKeyboard != null) { + mKeyboard = cachedKeyboard; + return; + } + final InputMethodSubtype subtype; + synchronized (mLock) { + subtype = mLocaleToSubtypeMap.get(newLocale); + } + if (subtype == null) { + return; + } + final EditorInfo editorInfo = new EditorInfo(); + editorInfo.inputType = InputType.TYPE_CLASS_TEXT; + final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( + mContext, editorInfo); + final Resources res = mContext.getResources(); + final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); + final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); + builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); + builder.setSubtype(subtype); + builder.setIsSpellChecker(false /* isSpellChecker */); + final KeyboardLayoutSet layoutSet = builder.build(); + mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); + } + + private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { + mDictionaryFacilitator.resetDictionaries(mContext, newlocale, + false /* useContactsDict */, false /* usePersonalizedDicts */, + false /* forceReloadMainDictionary */, null /* listener */); + mDictionaryFacilitator.waitForLoadingMainDictionary( + TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); + } + + /** + * Determine whether a word is a distracter to words in dictionaries. + * + * @param prevWordsInfo the information of previous words. + * @param testedWord the word that will be tested to see whether it is a distracter to words + * in dictionaries. + * @param locale the locale of word. + * @return true if testedWord is a distracter, otherwise false. + */ + @Override + public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, + final String testedWord, final Locale locale) { + if (locale == null) { + return false; + } + if (!locale.equals(mDictionaryFacilitator.getLocale())) { + synchronized (mLock) { + if (!mLocaleToSubtypeMap.containsKey(locale)) { + Log.e(TAG, "Locale " + locale + " is not enabled."); + // TODO: Investigate what we should do for disabled locales. + return false; + } + loadKeyboardForLocale(locale); + // Reset dictionaries for the locale. + try { + loadDictionariesForLocale(locale); + } catch (final InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", + e); + return false; + } + } + } + if (mKeyboard == null) { + return false; + } + final WordComposer composer = new WordComposer(); + final int[] codePoints = StringUtils.toCodePointArray(testedWord); + final int[] coordinates = mKeyboard.getCoordinates(codePoints); + composer.setComposingWord(codePoints, coordinates, prevWordsInfo); + + final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); + final String consideredWord = trailingSingleQuotesCount > 0 ? + testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : + testedWord; + final AsyncResultHolder holder = new AsyncResultHolder(); + final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + if (suggestedWords != null && suggestedWords.size() > 1) { + // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from + // the decoder is at index 1. + final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); + final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( + firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); + holder.set(hasStrongDistractor); + } + } + }; + mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), + true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, + null /* additionalFeaturesOptions */, 0 /* sessionId */, + SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + + return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + } +} diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java index 5631f97eb..ddbc8ac34 100644 --- a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java +++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java @@ -20,19 +20,19 @@ import java.util.Locale; import android.test.suitebuilder.annotation.LargeTest; -import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.utils.DistracterFilterUsingSuggestion; /** * Unit test for DistracterFilter */ @LargeTest public class DistracterFilterTest extends InputTestsBase { - private DistracterFilter mDistracterFilter; + private DistracterFilterUsingSuggestion mDistracterFilter; @Override protected void setUp() throws Exception { super.setUp(); - mDistracterFilter = new DistracterFilter(getContext()); + mDistracterFilter = new DistracterFilterUsingSuggestion(getContext()); mDistracterFilter.updateEnabledSubtypes(mLatinIME.getEnabledSubtypesForTest()); }