From 644a709a5fec65c3ac1c96f18af397458fac7658 Mon Sep 17 00:00:00 2001 From: Dan Zivkovic Date: Fri, 6 Feb 2015 15:07:16 -0800 Subject: [PATCH] Remove distracter filter from client. Bug 19296201. Change-Id: Ic834e5956347cd86a96bd14024c42ad8ee258659 --- .../latin/DictionaryFacilitatorProvider.java | 6 - .../latin/DictionaryFacilitator.java | 4 - .../latin/DictionaryFacilitatorImpl.java | 20 +- .../latin/ExpandableBinaryDictionary.java | 62 +--- .../android/inputmethod/latin/LatinIME.java | 4 +- .../UserHistoryDictionary.java | 8 +- .../latin/utils/DistracterFilter.java | 97 ------ ...terCheckingExactMatchesAndSuggestions.java | 324 ------------------ ...istracterFilterCheckingIsInDictionary.java | 64 ---- .../WordInputEventForPersonalization.java | 23 +- .../latin/BinaryDictionaryDecayingTests.java | 3 +- .../UserHistoryDictionaryTests.java | 4 +- .../UserHistoryDictionaryTestsHelper.java | 4 +- .../latin/utils/DistracterFilterTest.java | 225 ------------ 14 files changed, 25 insertions(+), 823 deletions(-) delete mode 100644 java/src/com/android/inputmethod/latin/utils/DistracterFilter.java delete mode 100644 java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java delete mode 100644 java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java delete mode 100644 tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java diff --git a/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java b/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java index 5489de135..205d6487a 100644 --- a/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java +++ b/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java @@ -16,8 +16,6 @@ package com.android.inputmethod.latin; -import android.content.Context; - /** * Factory for instantiating DictionaryFacilitator objects. */ @@ -25,8 +23,4 @@ public class DictionaryFacilitatorProvider { public static DictionaryFacilitator newDictionaryFacilitator() { return new DictionaryFacilitatorImpl(); } - - public static DictionaryFacilitator newDictionaryFacilitator(final Context context) { - return new DictionaryFacilitatorImpl(context); - } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 3e4cda47a..f15434208 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -18,7 +18,6 @@ package com.android.inputmethod.latin; import android.content.Context; import android.util.Pair; -import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.KeyboardLayout; @@ -28,7 +27,6 @@ import com.android.inputmethod.latin.utils.SuggestionResults; import java.io.File; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -90,8 +88,6 @@ public interface DictionaryFacilitator { void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); } - void updateEnabledSubtypes(final List enabledSubtypes); - // TODO: remove this, it's confusing with seamless multiple language switching void setIsMonolingualUser(final boolean isMonolingualUser); diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java index dd34faef8..48fe73be4 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java @@ -20,7 +20,6 @@ import android.content.Context; import android.text.TextUtils; import android.util.Log; import android.util.Pair; -import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.KeyboardLayout; @@ -29,9 +28,6 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.common.Constants; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; -import com.android.inputmethod.latin.utils.DistracterFilter; -import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions; -import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.SuggestionResults; @@ -42,7 +38,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -76,7 +71,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { private volatile CountDownLatch mLatchForWaitingLoadingMainDictionaries = new CountDownLatch(0); // To synchronize assigning mDictionaryGroup to ensure closing dictionaries. private final Object mLock = new Object(); - private final DistracterFilter mDistracterFilter; public static final Map> DICT_TYPE_TO_CLASS = new HashMap<>(); @@ -233,15 +227,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { } public DictionaryFacilitatorImpl() { - mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; - } - - public DictionaryFacilitatorImpl(final Context context) { - mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); - } - - public void updateEnabledSubtypes(final List enabledSubtypes) { - mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); } // TODO: remove this, it's confusing with seamless multiple language switching @@ -545,7 +530,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { dictionaryGroup.closeDict(dictType); } } - mDistracterFilter.close(); } @UsedForTesting @@ -659,9 +643,7 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { // We don't add words with 0-frequency (assuming they would be profanity etc.). final boolean isValid = maxFreq > 0; UserHistoryDictionary.addToDictionary(userHistoryDictionary, ngramContext, secondWord, - isValid, timeStampInSeconds, - new DistracterFilterCheckingIsInDictionary( - mDistracterFilter, userHistoryDictionary)); + isValid, timeStampInSeconds); } private void removeWord(final String dictName, final String word) { diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 87d46e226..413c5b8a1 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -31,7 +31,6 @@ import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CombinedFormatUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.WordInputEventForPersonalization; @@ -40,7 +39,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -57,7 +55,6 @@ import javax.annotation.Nullable; * * A class that extends this abstract class must have a static factory method named * getDictionary(Context context, Locale locale, File dictFile, String dictNamePrefix) - * @see DictionaryFacilitator#getSubDict(String,Context,Locale,File,String) */ abstract public class ExpandableBinaryDictionary extends Dictionary { private static final boolean DEBUG = false; @@ -172,33 +169,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static void asyncExecuteTaskWithLock(final Lock lock, final String executorName, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(lock, null /* preCheckTask */, executorName, task); - } - - private void asyncPreCheckAndExecuteTaskWithWriteLock( - final Callable preCheckTask, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(mLock.writeLock(), preCheckTask, - mDictName /* executorName */, task); - - } - - // Execute task with lock when the result of preCheckTask is true or preCheckTask is null. - private static void asyncPreCheckAndExecuteTaskWithLock(final Lock lock, - final Callable preCheckTask, final String executorName, final Runnable task) { - final String tag = TAG; ExecutorUtils.getExecutor(executorName).execute(new Runnable() { @Override public void run() { - if (preCheckTask != null) { - try { - if (!preCheckTask.call().booleanValue()) { - return; - } - } catch (final Exception e) { - Log.e(tag, "The pre check task throws an exception.", e); - return; - } - } lock.lock(); try { task.run(); @@ -305,17 +278,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - private void updateDictionaryWithWriteLockIfWordIsNotADistracter( - @Nonnull final Runnable updateTask, - @Nonnull final String word, @Nonnull final DistracterFilter distracterFilter) { + private void updateDictionaryWithWriteLock(@Nonnull final Runnable updateTask) { reloadDictionaryIfRequired(); - final Callable preCheckTask = new Callable() { - @Override - public Boolean call() throws Exception { - return !distracterFilter.isDistracterToWordsInDictionaries( - NgramContext.EMPTY_PREV_WORDS_INFO, word, mLocale); - } - }; final Runnable task = new Runnable() { @Override public void run() { @@ -326,23 +290,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { updateTask.run(); } }; - asyncPreCheckAndExecuteTaskWithWriteLock(preCheckTask, task); + asyncExecuteTaskWithWriteLock(task); } /** * Adds unigram information of a word to the dictionary. May overwrite an existing entry. */ - public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, + public void addUnigramEntry(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isPossiblyOffensive, final int timestamp, - @Nonnull final DistracterFilter distracterFilter) { - updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + final boolean isPossiblyOffensive, final int timestamp) { + updateDictionaryWithWriteLock(new Runnable() { @Override public void run() { addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, isNotAWord, isPossiblyOffensive, timestamp); } - }, word, distracterFilter); + }); } protected void addUnigramLocked(final String word, final int frequency, @@ -430,12 +393,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Update dictionary for the word with the ngramContext if the word is not a distracter. + * Update dictionary for the word with the ngramContext. */ - public void updateEntriesForWordWithCheckingDistracter(@Nonnull final NgramContext ngramContext, - final String word, final boolean isValidWord, final int count, final int timestamp, - @Nonnull final DistracterFilter distracterFilter) { - updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + public void updateEntriesForWord(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp) { + updateDictionaryWithWriteLock(new Runnable() { @Override public void run() { final BinaryDictionary binaryDictionary = getBinaryDictionary(); @@ -450,7 +412,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } } - }, word, distracterFilter); + }); } public interface UpdateEntriesForInputEventsCallback { @@ -653,7 +615,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary. Access is controlled on a per dictionary file basis. */ - private final void asyncReloadDictionary() { + private void asyncReloadDictionary() { final AtomicBoolean isReloading = mIsReloading; if (!isReloading.compareAndSet(false, true)) { return; diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 74ef6481a..c118e4aeb 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -128,7 +128,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Settings mSettings; private final DictionaryFacilitator mDictionaryFacilitator = - DictionaryFacilitatorProvider.newDictionaryFacilitator(this /* context */); + DictionaryFacilitatorProvider.newDictionaryFacilitator(); final InputLogic mInputLogic = new InputLogic(this /* LatinIME */, this /* SuggestionStripViewAccessor */, mDictionaryFacilitator); // We expect to have only one decoder in almost all cases, hence the default capacity of 1. @@ -608,8 +608,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mHandler.hasPendingReopenDictionaries()) { resetDictionaryFacilitator(locales); } - mDictionaryFacilitator.updateEnabledSubtypes(mRichImm.getMyEnabledInputMethodSubtypeList( - true /* allowsImplicitlySelectedSubtypes */)); refreshPersonalizationDictionarySession(currentSettingsValues); resetDictionaryFacilitatorIfNecessary(); mStatsUtilsManager.onLoadSettings(this /* context */, currentSettingsValues, diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 1d75a3098..b6286b203 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -25,7 +25,6 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.define.DecoderSpecificConstants; import com.android.inputmethod.latin.define.ProductionFlags; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.util.Locale; @@ -94,15 +93,14 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas * @param word the word the user inputted * @param isValid whether the word is valid or not * @param timestamp the timestamp when the word has been inputted - * @param distracterFilter the filter to check whether the word is a distracter */ public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, @Nonnull final NgramContext ngramContext, final String word, final boolean isValid, - final int timestamp, @Nonnull final DistracterFilter distracterFilter) { + final int timestamp) { if (word.length() > DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH) { return; } - userHistoryDictionary.updateEntriesForWordWithCheckingDistracter(ngramContext, word, - isValid, 1 /* count */, timestamp, distracterFilter); + userHistoryDictionary.updateEntriesForWord(ngramContext, word, + isValid, 1 /* count */, timestamp); } } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java deleted file mode 100644 index 525212c96..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.NgramContext; - -import java.util.List; -import java.util.Locale; - -import javax.annotation.Nonnull; - -public interface DistracterFilter { - /** - * Determine whether a word is a distracter to words in dictionaries. - * - * @param ngramContext the n-gram context - * @param testedWord the word that will be tested to see whether it is a distracter to words - * in dictionaries. - * @param locale the locale of word. - * @return true if testedWord is a distracter, otherwise false. - */ - public boolean isDistracterToWordsInDictionaries(final NgramContext ngramContext, - final String testedWord, final Locale locale); - - @UsedForTesting - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale); - - public void updateEnabledSubtypes(final List enabledSubtypes); - - public void close(); - - public static final class HandlingType { - private final static int REQUIRE_NO_SPECIAL_HANDLINGS = 0x0; - private final static int SHOULD_BE_LOWER_CASED = 0x1; - private final static int SHOULD_BE_HANDLED_AS_OOV = 0x2; - - public static int getHandlingType(final boolean shouldBeLowerCased, final boolean isOov) { - int wordHandlingType = HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS; - if (shouldBeLowerCased) { - wordHandlingType |= HandlingType.SHOULD_BE_LOWER_CASED; - } - if (isOov) { - wordHandlingType |= HandlingType.SHOULD_BE_HANDLED_AS_OOV; - } - return wordHandlingType; - } - - public static boolean shouldBeLowerCased(final int handlingType) { - return (handlingType & SHOULD_BE_LOWER_CASED) != 0; - } - - public static boolean shouldBeHandledAsOov(final int handlingType) { - return (handlingType & SHOULD_BE_HANDLED_AS_OOV) != 0; - } - } - - @Nonnull - public static final DistracterFilter EMPTY_DISTRACTER_FILTER = new DistracterFilter() { - @Override - public boolean isDistracterToWordsInDictionaries(NgramContext ngramContext, - String testedWord, Locale locale) { - return false; - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, - final String testedWord, final Locale locale) { - return HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS; - } - - @Override - public void close() { - } - - @Override - public void updateEnabledSubtypes(List enabledSubtypes) { - } - }; -} diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java deleted file mode 100644 index becf13fd9..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import android.content.Context; -import android.content.res.Resources; -import android.text.InputType; -import android.util.Log; -import android.util.LruCache; -import android.util.Pair; -import android.view.inputmethod.EditorInfo; -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardId; -import com.android.inputmethod.keyboard.KeyboardLayoutSet; -import com.android.inputmethod.latin.DictionaryFacilitator; -import com.android.inputmethod.latin.DictionaryFacilitatorLruCache; -import com.android.inputmethod.latin.NgramContext; -import com.android.inputmethod.latin.RichInputMethodSubtype; -import com.android.inputmethod.latin.SuggestedWords; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.common.StringUtils; -import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; - -/** - * This class is used to prevent distracters being added to personalization - * or user history dictionaries - */ -public class DistracterFilterCheckingExactMatchesAndSuggestions implements DistracterFilter { - private static final String TAG = - DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName(); - private static final boolean DEBUG = false; - - private static final int MAX_DISTRACTERS_CACHE_SIZE = 1024; - - private final Context mContext; - private final ConcurrentHashMap mLocaleToSubtypeCache; - private final ConcurrentHashMap mLocaleToKeyboardCache; - private final DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache; - // The key is a pair of a locale and a word. The value indicates the word is a distracter to - // words of the locale. - private final LruCache, Boolean> mDistractersCache; - private final Object mLock = new Object(); - - // If the score of the top suggestion exceeds this value, the tested word (e.g., - // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to - // words in dictionary. The greater the threshold is, the less likely the tested word would - // become a distracter, which means the tested word will be more likely to be added to - // the dictionary. - private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 0.4f; - - /** - * Create a DistracterFilter instance. - * - * @param context the context. - */ - public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context) { - mContext = context; - mLocaleToSubtypeCache = new ConcurrentHashMap<>(); - mLocaleToKeyboardCache = new ConcurrentHashMap<>(); - mDictionaryFacilitatorLruCache = new DictionaryFacilitatorLruCache( - context, "" /* dictionaryNamePrefix */); - mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE); - } - - @Override - public void close() { - mLocaleToSubtypeCache.clear(); - mLocaleToKeyboardCache.clear(); - mDictionaryFacilitatorLruCache.evictAll(); - // Don't clear mDistractersCache. - } - - @Override - public void updateEnabledSubtypes(final List enabledSubtypes) { - final Map newLocaleToSubtypeMap = new HashMap<>(); - if (enabledSubtypes != null) { - for (final InputMethodSubtype subtype : enabledSubtypes) { - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - if (newLocaleToSubtypeMap.containsKey(locale)) { - // Multiple subtypes are enabled for one locale. - // TODO: Investigate what we should do for this case. - continue; - } - newLocaleToSubtypeMap.put(locale, subtype); - } - } - if (mLocaleToSubtypeCache.equals(newLocaleToSubtypeMap)) { - // Enabled subtypes have not been changed. - return; - } - // Update subtype and keyboard map for locales that are in the current mapping. - for (final Locale locale: mLocaleToSubtypeCache.keySet()) { - if (newLocaleToSubtypeMap.containsKey(locale)) { - final InputMethodSubtype newSubtype = newLocaleToSubtypeMap.remove(locale); - if (newSubtype.equals(newLocaleToSubtypeMap.get(locale))) { - // Mapping has not been changed. - continue; - } - mLocaleToSubtypeCache.replace(locale, newSubtype); - } else { - mLocaleToSubtypeCache.remove(locale); - } - mLocaleToKeyboardCache.remove(locale); - } - // Add locales that are not in the current mapping. - mLocaleToSubtypeCache.putAll(newLocaleToSubtypeMap); - } - - private Keyboard getKeyboardForLocale(final Locale locale) { - final Keyboard cachedKeyboard = mLocaleToKeyboardCache.get(locale); - if (cachedKeyboard != null) { - return cachedKeyboard; - } - final InputMethodSubtype subtype = mLocaleToSubtypeCache.get(locale); - if (subtype == null) { - return null; - } - final EditorInfo editorInfo = new EditorInfo(); - editorInfo.inputType = InputType.TYPE_CLASS_TEXT; - final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( - mContext, editorInfo); - final Resources res = mContext.getResources(); - final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); - final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); - builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); - builder.setSubtype(new RichInputMethodSubtype(subtype)); - builder.setIsSpellChecker(false /* isSpellChecker */); - final KeyboardLayoutSet layoutSet = builder.build(); - final Keyboard newKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); - mLocaleToKeyboardCache.put(locale, newKeyboard); - return newKeyboard; - } - - /** - * Determine whether a word is a distracter to words in dictionaries. - * - * @param ngramContext the n-gram context. Not used for now. - * @param testedWord the word that will be tested to see whether it is a distracter to words - * in dictionaries. - * @param locale the locale of word. - * @return true if testedWord is a distracter, otherwise false. - */ - @Override - public boolean isDistracterToWordsInDictionaries(final NgramContext ngramContext, - final String testedWord, final Locale locale) { - if (locale == null) { - return false; - } - if (!mLocaleToSubtypeCache.containsKey(locale)) { - Log.e(TAG, "Locale " + locale + " is not enabled."); - // TODO: Investigate what we should do for disabled locales. - return false; - } - final DictionaryFacilitator dictionaryFacilitator = - mDictionaryFacilitatorLruCache.get(locale); - if (DEBUG) { - Log.d(TAG, "testedWord: " + testedWord); - } - final Pair cacheKey = new Pair<>(locale, testedWord); - final Boolean isCachedDistracter = mDistractersCache.get(cacheKey); - if (isCachedDistracter != null && isCachedDistracter) { - if (DEBUG) { - Log.d(TAG, "isDistracter: true (cache hit)"); - } - return true; - } - - final boolean isDistracterCheckedByGetMaxFreqencyOfExactMatches = - checkDistracterUsingMaxFreqencyOfExactMatches(dictionaryFacilitator, testedWord); - if (isDistracterCheckedByGetMaxFreqencyOfExactMatches) { - // Add the pair of locale and word to the cache. - mDistractersCache.put(cacheKey, Boolean.TRUE); - return true; - } - if (dictionaryFacilitator.isValidSuggestionWord(testedWord)) { - // Valid word is not a distracter. - if (DEBUG) { - Log.d(TAG, "isDistracter: false (valid word)"); - } - return false; - } - - final Keyboard keyboard = getKeyboardForLocale(locale); - final boolean isDistracterCheckedByGetSuggestion = - checkDistracterUsingGetSuggestions(dictionaryFacilitator, keyboard, testedWord); - if (isDistracterCheckedByGetSuggestion) { - // Add the pair of locale and word to the cache. - mDistractersCache.put(cacheKey, Boolean.TRUE); - return true; - } - return false; - } - - private static boolean checkDistracterUsingMaxFreqencyOfExactMatches( - final DictionaryFacilitator dictionaryFacilitator, final String testedWord) { - // The tested word is a distracter when there is a word that is exact matched to the tested - // word and its probability is higher than the tested word's probability. - final int perfectMatchFreq = dictionaryFacilitator.getFrequency(testedWord); - final int exactMatchFreq = dictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord); - final boolean isDistracter = perfectMatchFreq < exactMatchFreq; - if (DEBUG) { - Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq); - Log.d(TAG, "exactMatchFreq: " + exactMatchFreq); - Log.d(TAG, "isDistracter: " + isDistracter); - } - return isDistracter; - } - - private boolean checkDistracterUsingGetSuggestions( - final DictionaryFacilitator dictionaryFacilitator, final Keyboard keyboard, - final String testedWord) { - if (keyboard == null) { - return false; - } - final SettingsValuesForSuggestion settingsValuesForSuggestion = - new SettingsValuesForSuggestion(false /* blockPotentiallyOffensive */, - false /* spaceAwareGestureEnabled */); - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates = keyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates); - final SuggestionResults suggestionResults; - synchronized (mLock) { - suggestionResults = dictionaryFacilitator.getSuggestionResults(composer, - NgramContext.EMPTY_PREV_WORDS_INFO, - keyboard.getProximityInfo().getNativeProximityInfo(), - settingsValuesForSuggestion, 0 /* sessionId */, - SuggestedWords.INPUT_STYLE_TYPING, - keyboard.getKeyboardLayout()); - } - if (suggestionResults.isEmpty()) { - return false; - } - final SuggestedWordInfo firstSuggestion = suggestionResults.first(); - final boolean isDistracter = suggestionExceedsDistracterThreshold( - firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); - if (DEBUG) { - Log.d(TAG, "isDistracter: " + isDistracter); - } - return isDistracter; - } - - private static boolean suggestionExceedsDistracterThreshold(final SuggestedWordInfo suggestion, - final String consideredWord, final float distracterThreshold) { - if (suggestion == null) { - return false; - } - final int suggestionScore = suggestion.mScore; - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestion.mWord, suggestionScore); - if (DEBUG) { - Log.d(TAG, "normalizedScore: " + normalizedScore); - Log.d(TAG, "distracterThreshold: " + distracterThreshold); - } - if (normalizedScore > distracterThreshold) { - return true; - } - return false; - } - - private boolean shouldBeLowerCased(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - final DictionaryFacilitator dictionaryFacilitator = - mDictionaryFacilitatorLruCache.get(locale); - if (dictionaryFacilitator.isValidSuggestionWord(testedWord)) { - return false; - } - final String lowerCaseWord = testedWord.toLowerCase(locale); - if (testedWord.equals(lowerCaseWord)) { - return false; - } - if (dictionaryFacilitator.isValidSuggestionWord(lowerCaseWord)) { - return true; - } - if (StringUtils.getCapitalizationType(testedWord) == StringUtils.CAPITALIZE_FIRST - && !ngramContext.isValid()) { - // TODO: Check beginning-of-sentence. - return true; - } - return false; - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - // TODO: Use this method for user history dictionary. - if (testedWord == null|| locale == null) { - return HandlingType.getHandlingType(false /* shouldBeLowerCased */, false /* isOov */); - } - final boolean shouldBeLowerCased = shouldBeLowerCased(ngramContext, testedWord, locale); - final String caseModifiedWord = shouldBeLowerCased - ? testedWord.toLowerCase(locale) : testedWord; - final boolean isOov = !mDictionaryFacilitatorLruCache.get(locale).isValidSuggestionWord( - caseModifiedWord); - return HandlingType.getHandlingType(shouldBeLowerCased, isOov); - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java deleted file mode 100644 index 4c99fed9f..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.List; -import java.util.Locale; - -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.NgramContext; - -public class DistracterFilterCheckingIsInDictionary implements DistracterFilter { - private final DistracterFilter mDistracterFilter; - private final Dictionary mDictionary; - - public DistracterFilterCheckingIsInDictionary(final DistracterFilter distracterFilter, - final Dictionary dictionary) { - mDistracterFilter = distracterFilter; - mDictionary = dictionary; - } - - @Override - public boolean isDistracterToWordsInDictionaries(NgramContext ngramContext, - String testedWord, Locale locale) { - if (mDictionary.isInDictionary(testedWord)) { - // This filter treats entries that are already in the dictionary as non-distracters - // because they have passed the filtering in the past. - return false; - } - return mDistracterFilter.isDistracterToWordsInDictionaries( - ngramContext, testedWord, locale); - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - return mDistracterFilter.getWordHandlingType(ngramContext, testedWord, locale); - } - - @Override - public void updateEnabledSubtypes(List enabledSubtypes) { - // Do nothing. - } - - @Override - public void close() { - // Do nothing. - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java b/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java index e9a0e7a61..fc0a9cb6c 100644 --- a/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java +++ b/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java @@ -23,7 +23,6 @@ import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.common.StringUtils; import com.android.inputmethod.latin.define.DecoderSpecificConstants; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; -import com.android.inputmethod.latin.utils.DistracterFilter.HandlingType; import java.util.ArrayList; import java.util.List; @@ -41,17 +40,15 @@ public final class WordInputEventForPersonalization { new int[DecoderSpecificConstants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; public final boolean[] mIsPrevWordBeginningOfSentenceArray = new boolean[DecoderSpecificConstants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; - public final boolean mIsValid; // Time stamp in seconds. public final int mTimestamp; @UsedForTesting public WordInputEventForPersonalization(final CharSequence targetWord, - final NgramContext ngramContext, final boolean isValid, final int timestamp) { + final NgramContext ngramContext, final int timestamp) { mTargetWord = StringUtils.toCodePointArray(targetWord); mPrevWordsCount = ngramContext.getPrevWordCount(); ngramContext.outputToArray(mPrevWordArray, mIsPrevWordBeginningOfSentenceArray); - mIsValid = isValid; mTimestamp = timestamp; } @@ -59,8 +56,7 @@ public final class WordInputEventForPersonalization { // objects. public static ArrayList createInputEventFrom( final List tokens, final int timestamp, - final SpacingAndPunctuations spacingAndPunctuations, final Locale locale, - final DistracterFilter distracterFilter) { + final SpacingAndPunctuations spacingAndPunctuations, final Locale locale) { final ArrayList inputEvents = new ArrayList<>(); final int N = tokens.size(); NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; @@ -89,7 +85,7 @@ public final class WordInputEventForPersonalization { } final WordInputEventForPersonalization inputEvent = detectWhetherVaildWordOrNotAndGetInputEvent( - ngramContext, tempWord, timestamp, locale, distracterFilter); + ngramContext, tempWord, timestamp, locale); if (inputEvent == null) { continue; } @@ -101,19 +97,10 @@ public final class WordInputEventForPersonalization { private static WordInputEventForPersonalization detectWhetherVaildWordOrNotAndGetInputEvent( final NgramContext ngramContext, final String targetWord, final int timestamp, - final Locale locale, final DistracterFilter distracterFilter) { + final Locale locale) { if (locale == null) { return null; } - final int wordHandlingType = distracterFilter.getWordHandlingType(ngramContext, - targetWord, locale); - final String word = HandlingType.shouldBeLowerCased(wordHandlingType) ? - targetWord.toLowerCase(locale) : targetWord; - if (distracterFilter.isDistracterToWordsInDictionaries(ngramContext, targetWord, locale)) { - // The word is a distracter. - return null; - } - return new WordInputEventForPersonalization(word, ngramContext, - !HandlingType.shouldBeHandledAsOov(wordHandlingType), timestamp); + return new WordInputEventForPersonalization(targetWord, ngramContext, timestamp); } } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index cff489dd5..baac0883e 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -782,8 +782,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { int prevWordCount = 0; for (int i = 0; i < inputEvents.length; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); - inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext, - true /* isValid */, mCurrentTime); + inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext, mCurrentTime); unigrams.add(word); if (prevWordCount >= 2) { final Pair prevWordsPair = bigrams.get(bigrams.size() - 1); diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 6dddc971b..e1260f608 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -24,7 +24,6 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.util.List; @@ -252,8 +251,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { random); NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; for (final String word : words) { - UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime, - DistracterFilter.EMPTY_DISTRACTER_FILTER); + UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime); ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); dict.waitAllTasksForTests(); assertTrue(dict.isInDictionary(word)); diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java index d394c0faa..94105aa12 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java @@ -21,7 +21,6 @@ import android.content.Context; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.common.FileUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.io.FilenameFilter; @@ -101,8 +100,7 @@ public class UserHistoryDictionaryTestsHelper { final List words, final int timestamp) { NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; for (final String word : words) { - UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp, - DistracterFilter.EMPTY_DISTRACTER_FILTER); + UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp); ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); } } diff --git a/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java deleted file mode 100644 index 8360d53fb..000000000 --- a/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.ArrayList; -import java.util.Locale; - -import android.content.Context; -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.LargeTest; -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.latin.NgramContext; -import com.android.inputmethod.latin.RichInputMethodManager; -import com.android.inputmethod.latin.utils.DistracterFilter.HandlingType; - -/** - * Unit test for DistracterFilter - */ -@LargeTest -public class DistracterFilterTest extends AndroidTestCase { - private DistracterFilterCheckingExactMatchesAndSuggestions mDistracterFilter; - - @Override - protected void setUp() throws Exception { - super.setUp(); - final Context context = getContext(); - mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); - RichInputMethodManager.init(context); - final RichInputMethodManager richImm = RichInputMethodManager.getInstance(); - final ArrayList subtypes = new ArrayList<>(); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.US.toString(), "qwerty")); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.FRENCH.toString(), "azerty")); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.GERMAN.toString(), "qwertz")); - mDistracterFilter.updateEnabledSubtypes(subtypes); - } - - @Override - protected void tearDown() { - mDistracterFilter.close(); - } - - public void testIsDistracterToWordsInDictionaries() { - final NgramContext EMPTY_PREV_WORDS_INFO = NgramContext.EMPTY_PREV_WORDS_INFO; - - final Locale localeEnUs = new Locale("en", "US"); - String typedWord; - - typedWord = "Bill"; - // For this test case, we consider "Bill" is a distracter to "bill". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "nOt"; - // For this test case, we consider "nOt" is a distracter to "not". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "youre"; - // For this test case, we consider "youre" is a distracter to "you're". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "Banana"; - // For this test case, we consider "Banana" is a distracter to "banana". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "orange"; - // For this test case, we consider "orange" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "Orange"; - // For this test case, we consider "Orange" is a distracter to "orange". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "café"; - // For this test case, we consider "café" is a distracter to "cafe". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "cafe"; - // For this test case, we consider "cafe" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "I'll"; - // For this test case, we consider "I'll" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "ill"; - // For this test case, we consider "ill" is a distracter to "I'll" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "asdfd"; - // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries. - assertFalse( - mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thank"; - // For this test case, we consider "thank" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thabk"; - // For this test case, we consider "thabk" is a distracter to "thank" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thanks"; - // For this test case, we consider "thanks" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thabks"; - // For this test case, we consider "thabks" is a distracter to "thanks" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "think"; - // For this test case, we consider "think" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thibk"; - // For this test case, we consider "thibk" is a distracter to "think" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "tgis"; - // For this test case, we consider "tgis" is a distracter to "this" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - final Locale localeDeDe = new Locale("de"); - - typedWord = "fUEr"; - // For this test case, we consider "fUEr" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - typedWord = "fuer"; - // For this test case, we consider "fuer" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - typedWord = "fur"; - // For this test case, we consider "fur" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - final Locale localeFrFr = new Locale("fr"); - - typedWord = "a"; - // For this test case, we consider "a" is a distracter to "à". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "à"; - // For this test case, we consider "à" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "etre"; - // For this test case, we consider "etre" is a distracter to "être". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "États-unis"; - // For this test case, we consider "États-unis" is a distracter to "États-Unis". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "ÉtatsUnis"; - // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - } - - public void testGetWordHandlingType() { - final Locale localeEnUs = new Locale("en", "US"); - final NgramContext EMPTY_PREV_WORDS_INFO = NgramContext.EMPTY_PREV_WORDS_INFO; - int handlingType = 0; - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "this", localeEnUs); - assertFalse(HandlingType.shouldBeLowerCased(handlingType)); - assertFalse(HandlingType.shouldBeHandledAsOov(handlingType)); - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "This", localeEnUs); - assertTrue(HandlingType.shouldBeLowerCased(handlingType)); - assertFalse(HandlingType.shouldBeHandledAsOov(handlingType)); - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "thibk", localeEnUs); - assertFalse(HandlingType.shouldBeLowerCased(handlingType)); - assertTrue(HandlingType.shouldBeHandledAsOov(handlingType)); - } -}