LatinIME/java/src/com/android/inputmethod/latin/Suggest.java

/*
 * Copyright (C) 2008 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.inputmethod.latin;

import android.content.Context;
import android.text.TextUtils;
import android.util.Log;

import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.SettingsValues;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.StringUtils;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Locale;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

/**
 * This class loads a dictionary and provides a list of suggestions for a given sequence of
 * characters. This includes corrections and completions.
 */
// TODO: Separate dictionary operations from suggestions handling logic.
public final class Suggest {
    public static final String TAG = Suggest.class.getSimpleName();

    // Session id for
    // {@link #getSuggestedWords(WordComposer,String,ProximityInfo,boolean,int)}.
    // We are sharing the same ID between typing and gesture to save RAM footprint.
    public static final int SESSION_TYPING = 0;
    public static final int SESSION_GESTURE = 0;

    // TODO: rename this to CORRECTION_OFF
    public static final int CORRECTION_NONE = 0;
    // TODO: rename this to CORRECTION_ON
    public static final int CORRECTION_FULL = 1;

    // Close to -2**31
    private static final int SUPPRESS_SUGGEST_THRESHOLD = -2000000000;

    public static final int MAX_SUGGESTIONS = 18;

    public interface SuggestInitializationListener {
        public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable);
    }

    private static final boolean DBG = LatinImeLogger.sDBG;

    private final ConcurrentHashMap<String, Dictionary> mDictionaries =
            CollectionUtils.newConcurrentHashMap();
    private HashSet<String> mOnlyDictionarySetForDebug = null;
    private Dictionary mMainDictionary;
    private ContactsBinaryDictionary mContactsDictionary;
    private UserBinaryDictionary mUserDictionary;
    private UserHistoryDictionary mUserHistoryDictionary;
    private PersonalizationDictionary mPersonalizationDictionary;
    @UsedForTesting
    private boolean mIsCurrentlyWaitingForMainDictionary = false;

    private float mAutoCorrectionThreshold;

    // Locale used for upper- and title-casing words
    public final Locale mLocale;

    private final Context mContext;

    public Suggest(final Context context, final Locale locale, final SettingsValues settingsValues,
            final SuggestInitializationListener listener) {
        initAsynchronously(context, locale, listener);
        mLocale = locale;
        mContext = context;
        // initialize a debug flag for the personalization
        if (settingsValues.mUseOnlyPersonalizationDictionaryForDebug) {
            mOnlyDictionarySetForDebug = new HashSet<String>();
            mOnlyDictionarySetForDebug.add(Dictionary.TYPE_PERSONALIZATION);
        }
        setUserDictionary(new UserBinaryDictionary(context, locale));
    }

    @UsedForTesting
    Suggest(final Context context, final AssetFileAddress[] dictionaryList, final Locale locale) {
        final Dictionary mainDict = DictionaryFactory.createDictionaryForTest(dictionaryList,
                false /* useFullEditDistance */, locale);
        mLocale = locale;
        mContext = context;
        mMainDictionary = mainDict;
        addOrReplaceDictionaryInternal(Dictionary.TYPE_MAIN, mainDict);
    }

    private void initAsynchronously(final Context context, final Locale locale,
            final SuggestInitializationListener listener) {
        resetMainDict(context, locale, listener);
    }

    private void addOrReplaceDictionaryInternal(final String key, final Dictionary dict) {
        if (mOnlyDictionarySetForDebug != null && !mOnlyDictionarySetForDebug.contains(key)) {
            Log.w(TAG, "Ignore add " + key + " dictionary for debug.");
            return;
        }
        addOrReplaceDictionary(mDictionaries, key, dict);
    }

    private static void addOrReplaceDictionary(
            final ConcurrentHashMap<String, Dictionary> dictionaries,
            final String key, final Dictionary dict) {
        final Dictionary oldDict = (dict == null)
                ? dictionaries.remove(key)
                : dictionaries.put(key, dict);
        if (oldDict != null && dict != oldDict) {
            oldDict.close();
        }
    }

    public void resetMainDict(final Context context, final Locale locale,
            final SuggestInitializationListener listener) {
        mIsCurrentlyWaitingForMainDictionary = true;
        mMainDictionary = null;
        if (listener != null) {
            listener.onUpdateMainDictionaryAvailability(hasMainDictionary());
        }
        new Thread("InitializeBinaryDictionary") {
            @Override
            public void run() {
                final DictionaryCollection newMainDict =
                        DictionaryFactory.createMainDictionaryFromManager(context, locale);
                addOrReplaceDictionaryInternal(Dictionary.TYPE_MAIN, newMainDict);
                mMainDictionary = newMainDict;
                if (listener != null) {
                    listener.onUpdateMainDictionaryAvailability(hasMainDictionary());
                }
                mIsCurrentlyWaitingForMainDictionary = false;
            }
        }.start();
    }

    // The main dictionary could have been loaded asynchronously.  Don't cache the return value
    // of this method.
    public boolean hasMainDictionary() {
        return null != mMainDictionary && mMainDictionary.isInitialized();
    }

    @UsedForTesting
    public boolean isCurrentlyWaitingForMainDictionary() {
        return mIsCurrentlyWaitingForMainDictionary;
    }

    public Dictionary getMainDictionary() {
        return mMainDictionary;
    }

    /**
     * Sets an optional user dictionary resource to be loaded. The user dictionary is consulted
     * before the main dictionary, if set. This refers to the system-managed user dictionary.
     */
    @UsedForTesting
    public void setUserDictionary(final UserBinaryDictionary userDictionary) {
        mUserDictionary = userDictionary;
        addOrReplaceDictionaryInternal(Dictionary.TYPE_USER, userDictionary);
    }

    /**
     * Sets an optional contacts dictionary resource to be loaded. It is also possible to remove
     * the contacts dictionary by passing null to this method. In this case no contacts dictionary
     * won't be used.
     */
    @UsedForTesting
    public void setContactsDictionary(final ContactsBinaryDictionary contactsDictionary) {
        mContactsDictionary = contactsDictionary;
        addOrReplaceDictionaryInternal(Dictionary.TYPE_CONTACTS, contactsDictionary);
    }

    private void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) {
        mUserHistoryDictionary = userHistoryDictionary;
        addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary);
    }

    private void setPersonalizationDictionary(
            final PersonalizationDictionary personalizationDictionary) {
        mPersonalizationDictionary = personalizationDictionary;
        addOrReplaceDictionaryInternal(Dictionary.TYPE_PERSONALIZATION, personalizationDictionary);
    }

    /**
     * Set dictionaries that can be turned off according to the user settings.
     *
     * @param oldSuggest the instance having old dictionaries
     * @param settingsValues current SettingsValues
     */
    public void setAdditionalDictionaries(final Suggest oldSuggest,
            final SettingsValues settingsValues) {
        // Contacts dictionary
        resetContactsDictionary(null != oldSuggest ? oldSuggest.mContactsDictionary : null,
                settingsValues);
        // User history dictionary & Personalization dictionary
        resetPersonalizedDictionaries(oldSuggest, settingsValues);
    }

    /**
     * Set the user history dictionary and personalization dictionary according to the user
     * settings.
     *
     * @param oldSuggest the instance that has been used
     * @param settingsValues current settingsValues
     */
    // TODO: Consolidate resetPersonalizedDictionaries() and resetContactsDictionary(). Call up the
    // new method for each dictionary.
    private void resetPersonalizedDictionaries(final Suggest oldSuggest,
            final SettingsValues settingsValues) {
        final boolean shouldSetDictionaries = settingsValues.mUsePersonalizedDicts;

        final UserHistoryDictionary oldUserHistoryDictionary = (null == oldSuggest) ? null :
                oldSuggest.mUserHistoryDictionary;
        final PersonalizationDictionary oldPersonalizationDictionary = (null == oldSuggest) ? null :
                oldSuggest.mPersonalizationDictionary;
        final UserHistoryDictionary userHistoryDictionaryToUse;
        final PersonalizationDictionary personalizationDictionaryToUse;
        if (!shouldSetDictionaries) {
            userHistoryDictionaryToUse = null;
            personalizationDictionaryToUse = null;
        } else {
            if (null != oldUserHistoryDictionary
                    && oldUserHistoryDictionary.mLocale.equals(mLocale)) {
                userHistoryDictionaryToUse = oldUserHistoryDictionary;
            } else {
                userHistoryDictionaryToUse =
                        PersonalizationHelper.getUserHistoryDictionary(mContext, mLocale);
            }
            if (null != oldPersonalizationDictionary
                    && oldPersonalizationDictionary.mLocale.equals(mLocale)) {
                personalizationDictionaryToUse = oldPersonalizationDictionary;
            } else {
                personalizationDictionaryToUse =
                        PersonalizationHelper.getPersonalizationDictionary(mContext, mLocale);
            }
        }
        setUserHistoryDictionary(userHistoryDictionaryToUse);
        setPersonalizationDictionary(personalizationDictionaryToUse);
    }

    /**
     * Set the contacts dictionary according to the user settings.
     *
     * This method takes an optional contacts dictionary to use when the locale hasn't changed
     * since the contacts dictionary can be opened or closed as necessary depending on the settings.
     *
     * @param oldContactsDictionary an optional dictionary to use, or null
     * @param settingsValues current settingsValues
     */
    private void resetContactsDictionary(final ContactsBinaryDictionary oldContactsDictionary,
            final SettingsValues settingsValues) {
        final boolean shouldSetDictionary = settingsValues.mUseContactsDict;
        final ContactsBinaryDictionary dictionaryToUse;
        if (!shouldSetDictionary) {
            // Make sure the dictionary is closed. If it is already closed, this is a no-op,
            // so it's safe to call it anyways.
            if (null != oldContactsDictionary) oldContactsDictionary.close();
            dictionaryToUse = null;
        } else {
            if (null != oldContactsDictionary) {
                if (!oldContactsDictionary.mLocale.equals(mLocale)) {
                    // If the locale has changed then recreate the contacts dictionary. This
                    // allows locale dependent rules for handling bigram name predictions.
                    oldContactsDictionary.close();
                    dictionaryToUse = new ContactsBinaryDictionary(mContext, mLocale);
                } else {
                    // Make sure the old contacts dictionary is opened. If it is already open,
                    // this is a no-op, so it's safe to call it anyways.
                    oldContactsDictionary.reopen(mContext);
                    dictionaryToUse = oldContactsDictionary;
                }
            } else {
                dictionaryToUse = new ContactsBinaryDictionary(mContext, mLocale);
            }
        }
        setContactsDictionary(dictionaryToUse);
    }

    public boolean isUserDictionaryEnabled() {
        if (mUserDictionary == null) {
            return false;
        }
        return mUserDictionary.mEnabled;
    }

    public void addWordToUserDictionary(String word) {
        if (mUserDictionary == null) {
            return;
        }
        mUserDictionary.addWordToUserDictionary(word);
    }

    public String addToUserHistory(final WordComposer wordComposer, final String previousWord,
            final String suggestion) {
        if (mUserHistoryDictionary == null) {
            return null;
        }
        final String secondWord;
        if (wordComposer.wasAutoCapitalized() && !wordComposer.isMostlyCaps()) {
            secondWord = suggestion.toLowerCase(mLocale);
        } else {
            secondWord = suggestion;
        }
        // We demote unrecognized words (frequency < 0, below) by specifying them as "invalid".
        // We don't add words with 0-frequency (assuming they would be profanity etc.).
        final int maxFreq = getMaxFrequency(suggestion);
        if (maxFreq == 0) {
            return null;
        }
        final boolean isValid = maxFreq > 0;
        final int timeStamp = (int)TimeUnit.MILLISECONDS.toSeconds((System.currentTimeMillis()));
        mUserHistoryDictionary.addToDictionary(previousWord, secondWord, isValid, timeStamp);
        return previousWord;
    }

    public void cancelAddingUserHistory(final String previousWord, final String committedWord) {
        if (mUserHistoryDictionary != null) {
            mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord);
        }
    }

    public void setAutoCorrectionThreshold(float threshold) {
        mAutoCorrectionThreshold = threshold;
    }

    public interface OnGetSuggestedWordsCallback {
        public void onGetSuggestedWords(final SuggestedWords suggestedWords);
    }

    public void getSuggestedWords(final WordComposer wordComposer,
            final String prevWordForBigram, final ProximityInfo proximityInfo,
            final boolean blockOffensiveWords, final boolean isCorrectionEnabled,
            final int[] additionalFeaturesOptions, final int sessionId, final int sequenceNumber,
            final OnGetSuggestedWordsCallback callback) {
        LatinImeLogger.onStartSuggestion(prevWordForBigram);
        if (wordComposer.isBatchMode()) {
            getSuggestedWordsForBatchInput(wordComposer, prevWordForBigram, proximityInfo,
                    blockOffensiveWords, additionalFeaturesOptions, sessionId, sequenceNumber,
                    callback);
        } else {
            getSuggestedWordsForTypingInput(wordComposer, prevWordForBigram, proximityInfo,
                    blockOffensiveWords, isCorrectionEnabled, additionalFeaturesOptions,
                    sequenceNumber, callback);
        }
    }

    // Retrieves suggestions for the typing input
    // and calls the callback function with the suggestions.
    private void getSuggestedWordsForTypingInput(final WordComposer wordComposer,
            final String prevWordForBigram, final ProximityInfo proximityInfo,
            final boolean blockOffensiveWords, final boolean isCorrectionEnabled,
            final int[] additionalFeaturesOptions, final int sequenceNumber,
            final OnGetSuggestedWordsCallback callback) {
        final int trailingSingleQuotesCount = wordComposer.trailingSingleQuotesCount();
        final BoundedTreeSet suggestionsSet = new BoundedTreeSet(sSuggestedWordInfoComparator,
                MAX_SUGGESTIONS);

        final String typedWord = wordComposer.getTypedWord();
        final String consideredWord = trailingSingleQuotesCount > 0
                ? typedWord.substring(0, typedWord.length() - trailingSingleQuotesCount)
                : typedWord;
        LatinImeLogger.onAddSuggestedWord(typedWord, Dictionary.TYPE_USER_TYPED);

        final WordComposer wordComposerForLookup;
        if (trailingSingleQuotesCount > 0) {
            wordComposerForLookup = new WordComposer(wordComposer);
            for (int i = trailingSingleQuotesCount - 1; i >= 0; --i) {
                wordComposerForLookup.deleteLast();
            }
        } else {
            wordComposerForLookup = wordComposer;
        }

        for (final String key : mDictionaries.keySet()) {
            final Dictionary dictionary = mDictionaries.get(key);
            suggestionsSet.addAll(dictionary.getSuggestions(wordComposerForLookup,
                    prevWordForBigram, proximityInfo, blockOffensiveWords,
                    additionalFeaturesOptions));
        }

        final String whitelistedWord;
        if (suggestionsSet.isEmpty()) {
            whitelistedWord = null;
        } else if (SuggestedWordInfo.KIND_WHITELIST != suggestionsSet.first().mKind) {
            whitelistedWord = null;
        } else {
            whitelistedWord = suggestionsSet.first().mWord;
        }

        // The word can be auto-corrected if it has a whitelist entry that is not itself,
        // or if it's a 2+ characters non-word (i.e. it's not in the dictionary).
        final boolean allowsToBeAutoCorrected = (null != whitelistedWord
                && !whitelistedWord.equals(consideredWord))
                || (consideredWord.length() > 1
                        && !isValidWord(consideredWord, wordComposer.isFirstCharCapitalized()));

        final boolean hasAutoCorrection;
        // TODO: using isCorrectionEnabled here is not very good. It's probably useless, because
        // any attempt to do auto-correction is already shielded with a test for this flag; at the
        // same time, it feels wrong that the SuggestedWord object includes information about
        // the current settings. It may also be useful to know, when the setting is off, whether
        // the word *would* have been auto-corrected.
        if (!isCorrectionEnabled || !allowsToBeAutoCorrected || !wordComposer.isComposingWord()
                || suggestionsSet.isEmpty() || wordComposer.hasDigits()
                || wordComposer.isMostlyCaps() || wordComposer.isResumed() || !hasMainDictionary()
                || SuggestedWordInfo.KIND_SHORTCUT == suggestionsSet.first().mKind) {
            // If we don't have a main dictionary, we never want to auto-correct. The reason for
            // this is, the user may have a contact whose name happens to match a valid word in
            // their language, and it will unexpectedly auto-correct. For example, if the user
            // types in English with no dictionary and has a "Will" in their contact list, "will"
            // would always auto-correct to "Will" which is unwanted. Hence, no main dict => no
            // auto-correct.
            // Also, shortcuts should never auto-correct unless they are whitelist entries.
            // TODO: we may want to have shortcut-only entries auto-correct in the future.
            hasAutoCorrection = false;
        } else {
            hasAutoCorrection = AutoCorrectionUtils.suggestionExceedsAutoCorrectionThreshold(
                    suggestionsSet.first(), consideredWord, mAutoCorrectionThreshold);
        }

        final ArrayList<SuggestedWordInfo> suggestionsContainer =
                CollectionUtils.newArrayList(suggestionsSet);
        final int suggestionsCount = suggestionsContainer.size();
        final boolean isFirstCharCapitalized = wordComposer.isFirstCharCapitalized();
        final boolean isAllUpperCase = wordComposer.isAllUpperCase();
        if (isFirstCharCapitalized || isAllUpperCase || 0 != trailingSingleQuotesCount) {
            for (int i = 0; i < suggestionsCount; ++i) {
                final SuggestedWordInfo wordInfo = suggestionsContainer.get(i);
                final SuggestedWordInfo transformedWordInfo = getTransformedSuggestedWordInfo(
                        wordInfo, mLocale, isAllUpperCase, isFirstCharCapitalized,
                        trailingSingleQuotesCount);
                suggestionsContainer.set(i, transformedWordInfo);
            }
        }

        for (int i = 0; i < suggestionsCount; ++i) {
            final SuggestedWordInfo wordInfo = suggestionsContainer.get(i);
            LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(),
                    wordInfo.mSourceDict.mDictType);
        }

        if (!TextUtils.isEmpty(typedWord)) {
            suggestionsContainer.add(0, new SuggestedWordInfo(typedWord,
                    SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED,
                    Dictionary.DICTIONARY_USER_TYPED,
                    SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */,
                    SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */));
        }
        SuggestedWordInfo.removeDups(suggestionsContainer);

        final ArrayList<SuggestedWordInfo> suggestionsList;
        if (DBG && !suggestionsContainer.isEmpty()) {
            suggestionsList = getSuggestionsInfoListWithDebugInfo(typedWord, suggestionsContainer);
        } else {
            suggestionsList = suggestionsContainer;
        }

        callback.onGetSuggestedWords(new SuggestedWords(suggestionsList,
                // TODO: this first argument is lying. If this is a whitelisted word which is an
                // actual word, it says typedWordValid = false, which looks wrong. We should either
                // rename the attribute or change the value.
                !allowsToBeAutoCorrected /* typedWordValid */,
                hasAutoCorrection, /* willAutoCorrect */
                false /* isPunctuationSuggestions */,
                false /* isObsoleteSuggestions */,
                !wordComposer.isComposingWord() /* isPrediction */, sequenceNumber));
    }

    // Retrieves suggestions for the batch input
    // and calls the callback function with the suggestions.
    private void getSuggestedWordsForBatchInput(final WordComposer wordComposer,
            final String prevWordForBigram, final ProximityInfo proximityInfo,
            final boolean blockOffensiveWords, final int[] additionalFeaturesOptions,
            final int sessionId, final int sequenceNumber,
            final OnGetSuggestedWordsCallback callback) {
        final BoundedTreeSet suggestionsSet = new BoundedTreeSet(sSuggestedWordInfoComparator,
                MAX_SUGGESTIONS);

        // At second character typed, search the unigrams (scores being affected by bigrams)
        for (final String key : mDictionaries.keySet()) {
            final Dictionary dictionary = mDictionaries.get(key);
            suggestionsSet.addAll(dictionary.getSuggestionsWithSessionId(wordComposer,
                    prevWordForBigram, proximityInfo, blockOffensiveWords,
                    additionalFeaturesOptions, sessionId));
        }

        for (SuggestedWordInfo wordInfo : suggestionsSet) {
            LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType);
        }

        final ArrayList<SuggestedWordInfo> suggestionsContainer =
                CollectionUtils.newArrayList(suggestionsSet);
        final int suggestionsCount = suggestionsContainer.size();
        final boolean isFirstCharCapitalized = wordComposer.wasShiftedNoLock();
        final boolean isAllUpperCase = wordComposer.isAllUpperCase();
        if (isFirstCharCapitalized || isAllUpperCase) {
            for (int i = 0; i < suggestionsCount; ++i) {
                final SuggestedWordInfo wordInfo = suggestionsContainer.get(i);
                final SuggestedWordInfo transformedWordInfo = getTransformedSuggestedWordInfo(
                        wordInfo, mLocale, isAllUpperCase, isFirstCharCapitalized,
                        0 /* trailingSingleQuotesCount */);
                suggestionsContainer.set(i, transformedWordInfo);
            }
        }

        if (suggestionsContainer.size() > 1 && TextUtils.equals(suggestionsContainer.get(0).mWord,
                wordComposer.getRejectedBatchModeSuggestion())) {
            final SuggestedWordInfo rejected = suggestionsContainer.remove(0);
            suggestionsContainer.add(1, rejected);
        }
        SuggestedWordInfo.removeDups(suggestionsContainer);

        // For some reason some suggestions with MIN_VALUE are making their way here.
        // TODO: Find a more robust way to detect distractors.
        for (int i = suggestionsContainer.size() - 1; i >= 0; --i) {
            if (suggestionsContainer.get(i).mScore < SUPPRESS_SUGGEST_THRESHOLD) {
                suggestionsContainer.remove(i);
            }
        }

        // In the batch input mode, the most relevant suggested word should act as a "typed word"
        // (typedWordValid=true), not as an "auto correct word" (willAutoCorrect=false).
        callback.onGetSuggestedWords(new SuggestedWords(suggestionsContainer,
                true /* typedWordValid */,
                false /* willAutoCorrect */,
                false /* isPunctuationSuggestions */,
                false /* isObsoleteSuggestions */,
                false /* isPrediction */, sequenceNumber));
    }

    private static ArrayList<SuggestedWordInfo> getSuggestionsInfoListWithDebugInfo(
            final String typedWord, final ArrayList<SuggestedWordInfo> suggestions) {
        final SuggestedWordInfo typedWordInfo = suggestions.get(0);
        typedWordInfo.setDebugString("+");
        final int suggestionsSize = suggestions.size();
        final ArrayList<SuggestedWordInfo> suggestionsList =
                CollectionUtils.newArrayList(suggestionsSize);
        suggestionsList.add(typedWordInfo);
        // Note: i here is the index in mScores[], but the index in mSuggestions is one more
        // than i because we added the typed word to mSuggestions without touching mScores.
        for (int i = 0; i < suggestionsSize - 1; ++i) {
            final SuggestedWordInfo cur = suggestions.get(i + 1);
            final float normalizedScore = BinaryDictionary.calcNormalizedScore(
                    typedWord, cur.toString(), cur.mScore);
            final String scoreInfoString;
            if (normalizedScore > 0) {
                scoreInfoString = String.format(
                        Locale.ROOT, "%d (%4.2f), %s", cur.mScore, normalizedScore,
                        cur.mSourceDict.mDictType);
            } else {
                scoreInfoString = Integer.toString(cur.mScore);
            }
            cur.setDebugString(scoreInfoString);
            suggestionsList.add(cur);
        }
        return suggestionsList;
    }

    private static final class SuggestedWordInfoComparator
            implements Comparator<SuggestedWordInfo> {
        // This comparator ranks the word info with the higher frequency first. That's because
        // that's the order we want our elements in.
        @Override
        public int compare(final SuggestedWordInfo o1, final SuggestedWordInfo o2) {
            if (o1.mScore > o2.mScore) return -1;
            if (o1.mScore < o2.mScore) return 1;
            if (o1.mCodePointCount < o2.mCodePointCount) return -1;
            if (o1.mCodePointCount > o2.mCodePointCount) return 1;
            return o1.mWord.compareTo(o2.mWord);
        }
    }
    private static final SuggestedWordInfoComparator sSuggestedWordInfoComparator =
            new SuggestedWordInfoComparator();

    /* package for test */ static SuggestedWordInfo getTransformedSuggestedWordInfo(
            final SuggestedWordInfo wordInfo, final Locale locale, final boolean isAllUpperCase,
            final boolean isFirstCharCapitalized, final int trailingSingleQuotesCount) {
        final StringBuilder sb = new StringBuilder(wordInfo.mWord.length());
        if (isAllUpperCase) {
            sb.append(wordInfo.mWord.toUpperCase(locale));
        } else if (isFirstCharCapitalized) {
            sb.append(StringUtils.capitalizeFirstCodePoint(wordInfo.mWord, locale));
        } else {
            sb.append(wordInfo.mWord);
        }
        // Appending quotes is here to help people quote words. However, it's not helpful
        // when they type words with quotes toward the end like "it's" or "didn't", where
        // it's more likely the user missed the last character (or didn't type it yet).
        final int quotesToAppend = trailingSingleQuotesCount
                - (-1 == wordInfo.mWord.indexOf(Constants.CODE_SINGLE_QUOTE) ? 0 : 1);
        for (int i = quotesToAppend - 1; i >= 0; --i) {
            sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE);
        }
        return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind,
                wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord,
                wordInfo.mAutoCommitFirstWordConfidence);
    }

    public boolean isValidWord(final String word, final boolean ignoreCase) {
        if (TextUtils.isEmpty(word)) {
            return false;
        }
        final String lowerCasedWord = word.toLowerCase(mLocale);
        for (final String key : mDictionaries.keySet()) {
            final Dictionary dictionary = mDictionaries.get(key);
            // It's unclear how realistically 'dictionary' can be null, but the monkey is somehow
            // managing to get null in here. Presumably the language is changing to a language with
            // no main dictionary and the monkey manages to type a whole word before the thread
            // that reads the dictionary is started or something?
            // Ideally the passed map would come out of a {@link java.util.concurrent.Future} and
            // would be immutable once it's finished initializing, but concretely a null test is
            // probably good enough for the time being.
            if (null == dictionary) continue;
            if (dictionary.isValidWord(word)
                    || (ignoreCase && dictionary.isValidWord(lowerCasedWord))) {
                return true;
            }
        }
        return false;
    }

    private int getMaxFrequency(final String word) {
        if (TextUtils.isEmpty(word)) {
            return Dictionary.NOT_A_PROBABILITY;
        }
        int maxFreq = -1;
        for (final String key : mDictionaries.keySet()) {
            final Dictionary dictionary = mDictionaries.get(key);
            if (null == dictionary) continue;
            final int tempFreq = dictionary.getFrequency(word);
            if (tempFreq >= maxFreq) {
                maxFreq = tempFreq;
            }
        }
        return maxFreq;
    }

    public void close() {
        final HashSet<Dictionary> dictionaries = CollectionUtils.newHashSet();
        dictionaries.addAll(mDictionaries.values());
        for (final Dictionary dictionary : dictionaries) {
            dictionary.close();
        }
        mMainDictionary = null;
    }
}