Allow whitelist changes when no close word

When there isn't a close word in another language of the
current multi-language set, we allow whitelist entries
to take force even if we are not confident in the current
language.

Bug: 18063142
Bug: 18130489
Bug: 18132240
Bug: 18136721
Bug: 18200415
Change-Id: I044674ba7b70aa86ab2a48d2e4d53a1c8007b62c

java/res/values/config-auto-correction-thresholds.xml

@@ -34,6 +34,12 @@
         <item>floatNegativeInfinity</item>
     </string-array>
 
+    <!-- Chosen to be slightly less than the "aggressive" threshold. This is the threshold for
+         a mildly plausible suggestion given the input; if no "plausible" suggestion is present
+         for a language, it's a strong indicator the user is not typing in this language, so we
+         may be more forgiving of whitelist entries in another language. -->
+    <string name="plausibility_threshold" translatable="false">0.065</string>
+
     <!-- The index of the auto correction threshold values array. -->
     <string name="auto_correction_threshold_mode_index_off" translatable="false">0</string>
     <string name="auto_correction_threshold_mode_index_modest" translatable="false">1</string>

java/src/com/android/inputmethod/latin/LatinIME.java

@@ -708,6 +708,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
             mInputLogic.mSuggest.setAutoCorrectionThreshold(
                     settingsValues.mAutoCorrectionThreshold);
         }
+        mInputLogic.mSuggest.setPlausibilityThreshold(settingsValues.mPlausibilityThreshold);
     }
 
     /**
@@ -1007,6 +1008,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
                 suggest.setAutoCorrectionThreshold(
                         currentSettingsValues.mAutoCorrectionThreshold);
             }
+            suggest.setPlausibilityThreshold(currentSettingsValues.mPlausibilityThreshold);
 
             switcher.loadKeyboard(editorInfo, currentSettingsValues, getCurrentAutoCapsState(),
                     getCurrentRecapitalizeState());

java/src/com/android/inputmethod/latin/Suggest.java

@@ -65,15 +65,30 @@ public final class Suggest {
     }
 
     private float mAutoCorrectionThreshold;
+    private float mPlausibilityThreshold;
 
     public Suggest(final DictionaryFacilitator dictionaryFacilitator) {
         mDictionaryFacilitator = dictionaryFacilitator;
     }
 
+    /**
+     * Set the normalized-score threshold for a suggestion to be considered strong enough that we
+     * will auto-correct to this.
+     * @param threshold the threshold
+     */
     public void setAutoCorrectionThreshold(final float threshold) {
         mAutoCorrectionThreshold = threshold;
     }
 
+    /**
+     * Set the normalized-score threshold for what we consider a "plausible" suggestion, in
+     * the same dimension as the auto-correction threshold.
+     * @param threshold the threshold
+     */
+    public void setPlausibilityThreshold(final float threshold) {
+        mPlausibilityThreshold = threshold;
+    }
+
     public interface OnGetSuggestedWordsCallback {
         public void onGetSuggestedWords(final SuggestedWords suggestedWords);
     }
@@ -130,6 +145,18 @@ public final class Suggest {
         return firstSuggestedWordInfo;
     }
 
+    // Quality constants for dictionary match
+    // In increasing order of quality
+    // This source dictionary does not match the typed word.
+    private static final int QUALITY_NO_MATCH = 0;
+    // This source dictionary has a null locale, and the preferred locale is also null.
+    private static final int QUALITY_MATCH_NULL = 1;
+    // This source dictionary has a non-null locale different from the preferred locale. The
+    // preferred locale may be null : this is still better than MATCH_NULL.
+    private static final int QUALITY_MATCH_OTHER_LOCALE = 2;
+    // This source dictionary matches the preferred locale.
+    private static final int QUALITY_MATCH_PREFERRED_LOCALE = 3;
+
     // Retrieves suggestions for non-batch input (typing, recorrection, predictions...)
     // and calls the callback function with the suggestions.
     private void getSuggestedWordsForNonBatchInput(final WordComposer wordComposer,
@@ -154,20 +181,52 @@ public final class Suggest {
                         // For transforming suggestions that don't come for any dictionary, we
                         // use the currently most probable locale as it's our best bet.
                         mostProbableLocale);
-        @Nullable final Dictionary sourceDictionaryOfRemovedWord =
+
-                SuggestedWordInfo.removeDupsAndReturnSourceOfTypedWord(wordComposer.getTypedWord(),
+        boolean typedWordExistsInAnotherLanguage = false;
-                        mostProbableLocale /* preferredLocale */, suggestionsContainer);
+        int qualityOfFoundSourceDictionary = QUALITY_NO_MATCH;
+        @Nullable Dictionary sourceDictionaryOfRemovedWord = null;
+        for (final SuggestedWordInfo info : suggestionsContainer) {
+            // Search for the best dictionary, defined as the first one with the highest match
+            // quality we can find.
+            if (typedWordString.equals(info.mWord)) {
+                if (mostProbableLocale.equals(info.mSourceDict.mLocale)) {
+                    if (qualityOfFoundSourceDictionary < QUALITY_MATCH_PREFERRED_LOCALE) {
+                        // Use this source if the old match had lower quality than this match
+                        sourceDictionaryOfRemovedWord = info.mSourceDict;
+                        qualityOfFoundSourceDictionary = QUALITY_MATCH_PREFERRED_LOCALE;
+                    }
+                } else {
+                    final int matchQuality = (null == info.mSourceDict.mLocale)
+                            ? QUALITY_MATCH_NULL : QUALITY_MATCH_OTHER_LOCALE;
+                    if (qualityOfFoundSourceDictionary < matchQuality) {
+                        // Use this source if the old match had lower quality than this match
+                        sourceDictionaryOfRemovedWord = info.mSourceDict;
+                        qualityOfFoundSourceDictionary = matchQuality;
+                    }
+                    typedWordExistsInAnotherLanguage = true;
+                }
+            }
+        }
+
+        SuggestedWordInfo.removeDups(typedWordString, suggestionsContainer);
 
         final SuggestedWordInfo whitelistedWordInfo =
                 getWhitelistedWordInfoOrNull(suggestionsContainer);
         final String whitelistedWord;
         if (null != whitelistedWordInfo &&
-                mDictionaryFacilitator.isConfidentAboutCurrentLanguageBeing(
+                (mDictionaryFacilitator.isConfidentAboutCurrentLanguageBeing(
-                        whitelistedWordInfo.mSourceDict.mLocale)) {
+                        whitelistedWordInfo.mSourceDict.mLocale)
+                || (!typedWordExistsInAnotherLanguage
+                        && !hasPlausibleCandidateInAnyOtherLanguage(suggestionsContainer,
+                                consideredWord, whitelistedWordInfo)))) {
+            // We'll use the whitelist candidate if we are confident the user is typing in the
+            // language of the dictionary it's coming from, or if there is no plausible candidate
+            // coming from another language.
             whitelistedWord = whitelistedWordInfo.mWord;
         } else {
-            // Even if we have a whitelist candidate, we don't use it unless we are confident
+            // If on the contrary we are not confident in the current language and we have
-            // the user is typing in the language this whitelist candidate comes from.
+            // at least a plausible candidate in any other language, then we don't use this
+            // whitelist candidate.
             whitelistedWord = null;
         }
         final boolean resultsArePredictions = !wordComposer.isComposingWord();
@@ -211,7 +270,7 @@ public final class Suggest {
             hasAutoCorrection = false;
         } else {
             final SuggestedWordInfo firstSuggestion = suggestionResults.first();
-            if (!AutoCorrectionUtils.suggestionExceedsAutoCorrectionThreshold(
+            if (!AutoCorrectionUtils.suggestionExceedsThreshold(
                     firstSuggestion, consideredWord, mAutoCorrectionThreshold)) {
                 // Score is too low for autocorrect
                 hasAutoCorrection = false;
@@ -260,6 +319,20 @@ public final class Suggest {
                 false /* isObsoleteSuggestions */, inputStyle, sequenceNumber));
     }
 
+    private boolean hasPlausibleCandidateInAnyOtherLanguage(
+            final ArrayList<SuggestedWordInfo> suggestionsContainer, final String consideredWord,
+            final SuggestedWordInfo whitelistedWordInfo) {
+        for (final SuggestedWordInfo info : suggestionsContainer) {
+            if (whitelistedWordInfo.mSourceDict.mLocale.equals(info.mSourceDict.mLocale)) {
+                continue;
+            }
+            return AutoCorrectionUtils.suggestionExceedsThreshold(info, consideredWord,
+                    mPlausibilityThreshold);
+        }
+        // No candidate in another language
+        return false;
+    }
+
     // Retrieves suggestions for the batch input
     // and calls the callback function with the suggestions.
     private void getSuggestedWordsForBatchInput(final WordComposer wordComposer,
@@ -293,8 +366,7 @@ public final class Suggest {
             final SuggestedWordInfo rejected = suggestionsContainer.remove(0);
             suggestionsContainer.add(1, rejected);
         }
-        SuggestedWordInfo.removeDupsAndReturnSourceOfTypedWord(null /* typedWord */,
+        SuggestedWordInfo.removeDups(null /* typedWord */, suggestionsContainer);
-                null /* preferredLocale */, suggestionsContainer);
 
         // For some reason some suggestions with MIN_VALUE are making their way here.
         // TODO: Find a more robust way to detect distracters.

java/src/com/android/inputmethod/latin/SuggestedWords.java

@@ -356,53 +356,30 @@ public class SuggestedWords {
         }
 
         // This will always remove the higher index if a duplicate is found.
-        // Returns null if the typed word is not found. Always return the dictionary for the
+        public static void removeDups(@Nullable final String typedWord,
-        // highest suggestion matching the locale if found, otherwise return the dictionary for
-        // the highest suggestion.
-        @Nullable
-        public static Dictionary removeDupsAndReturnSourceOfTypedWord(
-                @Nullable final String typedWord,
-                @Nullable final Locale preferredLocale,
                 @Nonnull ArrayList<SuggestedWordInfo> candidates) {
             if (candidates.isEmpty()) {
-                return null;
+                return;
             }
-            final Dictionary sourceDictionaryOfTypedWord;
             if (!TextUtils.isEmpty(typedWord)) {
-                sourceDictionaryOfTypedWord =
+                removeSuggestedWordInfoFromList(typedWord, candidates, -1 /* startIndexExclusive */);
-                        removeSuggestedWordInfoFromListAndReturnSourceDictionary(typedWord,
-                                preferredLocale, candidates, -1 /* startIndexExclusive */);
-            } else {
-                sourceDictionaryOfTypedWord = null;
             }
             for (int i = 0; i < candidates.size(); ++i) {
-                removeSuggestedWordInfoFromListAndReturnSourceDictionary(candidates.get(i).mWord,
+                removeSuggestedWordInfoFromList(candidates.get(i).mWord, candidates,
-                        null /* preferredLocale */, candidates, i /* startIndexExclusive */);
+                        i /* startIndexExclusive */);
             }
-            return sourceDictionaryOfTypedWord;
         }
 
-        @Nullable
+        private static void removeSuggestedWordInfoFromList(
-        private static Dictionary removeSuggestedWordInfoFromListAndReturnSourceDictionary(
+                @Nonnull final String word, @Nonnull final ArrayList<SuggestedWordInfo> candidates,
-                @Nonnull final String word, @Nullable final Locale preferredLocale,
-                @Nonnull final ArrayList<SuggestedWordInfo> candidates,
                 final int startIndexExclusive) {
-            Dictionary sourceDictionaryOfTypedWord = null;
             for (int i = startIndexExclusive + 1; i < candidates.size(); ++i) {
                 final SuggestedWordInfo previous = candidates.get(i);
                 if (word.equals(previous.mWord)) {
-                    if (null == sourceDictionaryOfTypedWord
-                            || (null != preferredLocale
-                                    && preferredLocale.equals(previous.mSourceDict.mLocale))) {
-                        if (Dictionary.TYPE_USER_HISTORY != previous.mSourceDict.mDictType) {
-                            sourceDictionaryOfTypedWord = previous.mSourceDict;
-                        }
-                    }
                     candidates.remove(i);
                     --i;
                 }
             }
-            return sourceDictionaryOfTypedWord;
         }
     }
 

java/src/com/android/inputmethod/latin/settings/Settings.java

@@ -238,6 +238,10 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang
         return !currentAutoCorrectionSetting.equals(autoCorrectionOff);
     }
 
+    public static float readPlausibilityThreshold(final Resources res) {
+        return Float.parseFloat(res.getString(R.string.plausibility_threshold));
+    }
+
     public static boolean readBlockPotentiallyOffensive(final SharedPreferences prefs,
             final Resources res) {
         return prefs.getBoolean(PREF_BLOCK_POTENTIALLY_OFFENSIVE,

java/src/com/android/inputmethod/latin/settings/SettingsValues.java

@@ -96,6 +96,7 @@ public class SettingsValues {
     public final int mKeyPreviewPopupDismissDelay;
     private final boolean mAutoCorrectEnabled;
     public final float mAutoCorrectionThreshold;
+    public final float mPlausibilityThreshold;
     public final boolean mAutoCorrectionEnabledPerUserSettings;
     private final boolean mSuggestionsEnabledPerUserSettings;
     private final AsyncResultHolder<AppWorkaroundsUtils> mAppWorkarounds;
@@ -172,6 +173,7 @@ public class SettingsValues {
                 Settings.PREF_ENABLE_EMOJI_ALT_PHYSICAL_KEY, true);
         mAutoCorrectionThreshold = readAutoCorrectionThreshold(res,
                 autoCorrectionThresholdRawValue);
+        mPlausibilityThreshold = Settings.readPlausibilityThreshold(res);
         mGestureInputEnabled = Settings.readGestureInputEnabled(prefs, res);
         mGestureTrailEnabled = prefs.getBoolean(Settings.PREF_GESTURE_PREVIEW_TRAIL, true);
         mGestureFloatingPreviewTextEnabled = !mInputAttributes.mDisableGestureFloatingPreviewText

java/src/com/android/inputmethod/latin/utils/AutoCorrectionUtils.java

@@ -29,9 +29,8 @@ public final class AutoCorrectionUtils {
         // Purely static class: can't instantiate.
     }
 
-    public static boolean suggestionExceedsAutoCorrectionThreshold(
+    public static boolean suggestionExceedsThreshold(final SuggestedWordInfo suggestion,
-            final SuggestedWordInfo suggestion, final String consideredWord,
+            final String consideredWord, final float threshold) {
-            final float autoCorrectionThreshold) {
         if (null != suggestion) {
             // Shortlist a whitelisted word
             if (suggestion.isKindOf(SuggestedWordInfo.KIND_WHITELIST)) {
@@ -45,11 +44,11 @@ public final class AutoCorrectionUtils {
             if (DBG) {
                 Log.d(TAG, "Normalized " + consideredWord + "," + suggestion + ","
                         + autoCorrectionSuggestionScore + ", " + normalizedScore
-                        + "(" + autoCorrectionThreshold + ")");
+                        + "(" + threshold + ")");
             }
-            if (normalizedScore >= autoCorrectionThreshold) {
+            if (normalizedScore >= threshold) {
                 if (DBG) {
-                    Log.d(TAG, "Auto corrected by S-threshold.");
+                    Log.d(TAG, "Exceeds threshold.");
                 }
                 return true;
             }