nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am ab6f3b36: Change DictionaryFacilitator.java to an interface, create its implementation DictionaryFacilitatorImpl.java and add a java-overridable factory DictionaryFacilitatorProvider.java used to create a DictionaryFacilitator. 

* commit 'ab6f3b36d0303bc4cc1ad0fbbc72a64ca2df0eb2':
  Change DictionaryFacilitator.java to an interface, create its implementation DictionaryFacilitatorImpl.java and add a java-overridable factory DictionaryFacilitatorProvider.java used to create a DictionaryFacilitator.
main
Mario Tanev 2015-01-27 21:46:51 +00:00 committed by Android Git Automerger
parent a8278c9474 ab6f3b36d0
commit d7409c2e1f
8 changed files with 1045 additions and 840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
import android.content.Context;
/**
* Factory for instantiating DictionaryFacilitator objects.
*/
public class DictionaryFacilitatorProvider {
public static DictionaryFacilitator newDictionaryFacilitator() {
return new DictionaryFacilitatorImpl();
}
public static DictionaryFacilitator newDictionaryFacilitator(final Context context) {
return new DictionaryFacilitatorImpl(context);
}
}

886
java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
View File

@ -17,272 +17,58 @@
package com.android.inputmethod.latin; package com.android.inputmethod.latin;
import android.content.Context; import android.content.Context;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import android.view.inputmethod.InputMethodSubtype; import android.view.inputmethod.InputMethodSubtype;
import android.util.Pair;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.UpdateEntriesForInputEventsCallback; import com.android.inputmethod.latin.ExpandableBinaryDictionary.UpdateEntriesForInputEventsCallback;
import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.common.Constants;
import com.android.inputmethod.latin.personalization.ContextualDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; import com.android.inputmethod.latin.personalization.PersonalizationDataChunk;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.SuggestionResults; import com.android.inputmethod.latin.utils.SuggestionResults;
import java.io.File; import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
/** /**
* Facilitates interaction with different kinds of dictionaries. Provides APIs * Interface that facilitates interaction with different kinds of dictionaries. Provides APIs to
* to instantiate and select the correct dictionaries (based on language or account), * instantiate and select the correct dictionaries (based on language or account), update entries
* update entries and fetch suggestions. * and fetch suggestions. Currently AndroidSpellCheckerService and LatinIME both use
* * DictionaryFacilitator as a client for interacting with dictionaries.
* Currently AndroidSpellCheckerService and LatinIME both use DictionaryFacilitator as
* a client for interacting with dictionaries.
*/ */
public class DictionaryFacilitator { public interface DictionaryFacilitator {
// TODO: Consolidate dictionaries in native code.
public static final String TAG = DictionaryFacilitator.class.getSimpleName();
// HACK: This threshold is being used when adding a capitalized entry in the User History
// dictionary.
private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140;
// How many words we need to type in a row ({@see mConfidenceInMostProbableLanguage}) to
// declare we are confident the user is typing in the most probable language.
private static final int CONFIDENCE_THRESHOLD = 3;
private DictionaryGroup[] mDictionaryGroups = new DictionaryGroup[] { new DictionaryGroup() };
private DictionaryGroup mMostProbableDictionaryGroup = mDictionaryGroups[0];
private boolean mIsUserDictEnabled = false;
private volatile CountDownLatch mLatchForWaitingLoadingMainDictionaries = new CountDownLatch(0);
// To synchronize assigning mDictionaryGroup to ensure closing dictionaries.
private final Object mLock = new Object();
private final DistracterFilter mDistracterFilter;
private final PersonalizationHelperForDictionaryFacilitator mPersonalizationHelper;
private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
new String[] {
Dictionary.TYPE_MAIN,
Dictionary.TYPE_USER_HISTORY,
Dictionary.TYPE_PERSONALIZATION,
Dictionary.TYPE_USER,
Dictionary.TYPE_CONTACTS,
Dictionary.TYPE_CONTEXTUAL
};
public static final Map<String, Class<? extends ExpandableBinaryDictionary>>
DICT_TYPE_TO_CLASS = new HashMap<>();
static {
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER_HISTORY, UserHistoryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_PERSONALIZATION, PersonalizationDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER, UserBinaryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTACTS, ContactsBinaryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTEXTUAL, ContextualDictionary.class);
}
private static final String DICT_FACTORY_METHOD_NAME = "getDictionary";
private static final Class<?>[] DICT_FACTORY_METHOD_ARG_TYPES =
new Class[] { Context.class, Locale.class, File.class, String.class, String.class };
private static final String[] SUB_DICT_TYPES =
Arrays.copyOfRange(DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS, 1 /* start */,
DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS.length);
/** /**
* Returns whether this facilitator is exactly for this list of locales. * Returns whether this facilitator is exactly for this list of locales.
* *
* @param locales the list of locales to test against * @param locales the list of locales to test against
*/ */
public boolean isForLocales(final Locale[] locales) { boolean isForLocales(final Locale[] locales);
if (locales.length != mDictionaryGroups.length) {
return false;
}
for (final Locale locale : locales) {
boolean found = false;
for (final DictionaryGroup group : mDictionaryGroups) {
if (locale.equals(group.mLocale)) {
found = true;
break;
}
}
if (!found) {
return false;
}
}
return true;
}
/** /**
* Returns whether this facilitator is exactly for this account. * Returns whether this facilitator is exactly for this account.
* *
* @param account the account to test against. * @param account the account to test against.
*/ */
public boolean isForAccount(@Nullable final String account) { boolean isForAccount(@Nullable final String account);
for (final DictionaryGroup group : mDictionaryGroups) {
if (!TextUtils.equals(group.mAccount, account)) { interface DictionaryInitializationListener {
return false; void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable);
}
}
return true;
} }
/** void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes);
* A group of dictionaries that work together for a single language.
*/
private static class DictionaryGroup {
// TODO: Add null analysis annotations.
// TODO: Run evaluation to determine a reasonable value for these constants. The current
// values are ad-hoc and chosen without any particular care or methodology.
public static final float WEIGHT_FOR_MOST_PROBABLE_LANGUAGE = 1.0f;
public static final float WEIGHT_FOR_GESTURING_IN_NOT_MOST_PROBABLE_LANGUAGE = 0.95f;
public static final float WEIGHT_FOR_TYPING_IN_NOT_MOST_PROBABLE_LANGUAGE = 0.6f;
/**
* The locale associated with the dictionary group.
*/
@Nullable public final Locale mLocale;
/**
* The user account associated with the dictionary group.
*/
@Nullable public final String mAccount;
@Nullable private Dictionary mMainDict;
// Confidence that the most probable language is actually the language the user is
// typing in. For now, this is simply the number of times a word from this language
// has been committed in a row.
private int mConfidence = 0;
public float mWeightForTypingInLocale = WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
public float mWeightForGesturingInLocale = WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap =
new ConcurrentHashMap<>();
public DictionaryGroup() {
this(null /* locale */, null /* mainDict */, null /* account */,
Collections.<String, ExpandableBinaryDictionary>emptyMap() /* subDicts */);
}
public DictionaryGroup(@Nullable final Locale locale,
@Nullable final Dictionary mainDict,
@Nullable final String account,
final Map<String, ExpandableBinaryDictionary> subDicts) {
mLocale = locale;
mAccount = account;
// The main dictionary can be asynchronously loaded.
setMainDict(mainDict);
for (final Map.Entry<String, ExpandableBinaryDictionary> entry : subDicts.entrySet()) {
setSubDict(entry.getKey(), entry.getValue());
}
}
private void setSubDict(final String dictType, final ExpandableBinaryDictionary dict) {
if (dict != null) {
mSubDictMap.put(dictType, dict);
}
}
public void setMainDict(final Dictionary mainDict) {
// Close old dictionary if exists. Main dictionary can be assigned multiple times.
final Dictionary oldDict = mMainDict;
mMainDict = mainDict;
if (oldDict != null && mainDict != oldDict) {
oldDict.close();
}
}
public Dictionary getDict(final String dictType) {
if (Dictionary.TYPE_MAIN.equals(dictType)) {
return mMainDict;
}
return getSubDict(dictType);
}
public ExpandableBinaryDictionary getSubDict(final String dictType) {
return mSubDictMap.get(dictType);
}
public boolean hasDict(final String dictType, @Nullable final String account) {
if (Dictionary.TYPE_MAIN.equals(dictType)) {
return mMainDict != null;
}
if (Dictionary.TYPE_USER_HISTORY.equals(dictType) &&
!TextUtils.equals(account, mAccount)) {
// If the dictionary type is user history, & if the account doesn't match,
// return immediately. If the account matches, continue looking it up in the
// sub dictionary map.
return false;
}
return mSubDictMap.containsKey(dictType);
}
public void closeDict(final String dictType) {
final Dictionary dict;
if (Dictionary.TYPE_MAIN.equals(dictType)) {
dict = mMainDict;
} else {
dict = mSubDictMap.remove(dictType);
}
if (dict != null) {
dict.close();
}
}
}
public interface DictionaryInitializationListener {
public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable);
}
public DictionaryFacilitator() {
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
mPersonalizationHelper = null;
}
public DictionaryFacilitator(final Context context) {
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);
mPersonalizationHelper =
new PersonalizationHelperForDictionaryFacilitator(context, mDistracterFilter);
}
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
mDistracterFilter.updateEnabledSubtypes(enabledSubtypes);
mPersonalizationHelper.updateEnabledSubtypes(enabledSubtypes);
}
// TODO: remove this, it's confusing with seamless multiple language switching // TODO: remove this, it's confusing with seamless multiple language switching
public void setIsMonolingualUser(final boolean isMonolingualUser) { void setIsMonolingualUser(final boolean isMonolingualUser);
mPersonalizationHelper.setIsMonolingualUser(isMonolingualUser);
}
public boolean isActive() { boolean isActive();
return null != mDictionaryGroups[0].mLocale;
}
/** /**
* Returns the most probable locale among all currently active locales. BE CAREFUL using this. * Returns the most probable locale among all currently active locales. BE CAREFUL using this.
@ -292,660 +78,96 @@ public class DictionaryFacilitator {
* string. * string.
* @return the most probable locale * @return the most probable locale
*/ */
public Locale getMostProbableLocale() { Locale getMostProbableLocale();
return getDictionaryGroupForMostProbableLanguage().mLocale;
}
public Locale[] getLocales() { Locale[] getLocales();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
final Locale[] locales = new Locale[dictionaryGroups.length];
for (int i = 0; i < dictionaryGroups.length; ++i) {
locales[i] = dictionaryGroups[i].mLocale;
}
return locales;
}
private DictionaryGroup getDictionaryGroupForMostProbableLanguage() { void switchMostProbableLanguage(@Nullable final Locale locale);
return mMostProbableDictionaryGroup;
}
public void switchMostProbableLanguage(@Nullable final Locale locale) { boolean isConfidentAboutCurrentLanguageBeing(final Locale mLocale);
if (null == locale) {
// In many cases, there is no locale to a committed word. For example, a typed word
// that is in none of the currently active dictionaries but still does not
// auto-correct to anything has no locale. In this case we simply do not change
// the most probable language and do not touch confidence.
return;
}
final DictionaryGroup newMostProbableDictionaryGroup =
findDictionaryGroupWithLocale(mDictionaryGroups, locale);
if (null == newMostProbableDictionaryGroup) {
// It seems this may happen as a race condition; pressing the globe key and space
// in quick succession could commit a word out of a dictionary that's not in the
// facilitator any more. In this case, just not changing things is fine.
return;
}
if (newMostProbableDictionaryGroup == mMostProbableDictionaryGroup) {
++newMostProbableDictionaryGroup.mConfidence;
} else {
mMostProbableDictionaryGroup.mWeightForTypingInLocale =
DictionaryGroup.WEIGHT_FOR_TYPING_IN_NOT_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup.mWeightForGesturingInLocale =
DictionaryGroup.WEIGHT_FOR_GESTURING_IN_NOT_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup.mConfidence = 0;
newMostProbableDictionaryGroup.mWeightForTypingInLocale =
DictionaryGroup.WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
newMostProbableDictionaryGroup.mWeightForGesturingInLocale =
DictionaryGroup.WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup = newMostProbableDictionaryGroup;
}
}
public boolean isConfidentAboutCurrentLanguageBeing(final Locale mLocale) { void resetDictionaries(final Context context, final Locale[] newLocales,
final DictionaryGroup mostProbableDictionaryGroup = mMostProbableDictionaryGroup;
if (!mostProbableDictionaryGroup.mLocale.equals(mLocale)) {
return false;
}
if (mDictionaryGroups.length <= 1) {
return true;
}
return mostProbableDictionaryGroup.mConfidence >= CONFIDENCE_THRESHOLD;
}
@Nullable
private static ExpandableBinaryDictionary getSubDict(final String dictType,
final Context context, final Locale locale, final File dictFile,
final String dictNamePrefix, @Nullable final String account) {
final Class<? extends ExpandableBinaryDictionary> dictClass =
DICT_TYPE_TO_CLASS.get(dictType);
if (dictClass == null) {
return null;
}
try {
final Method factoryMethod = dictClass.getMethod(DICT_FACTORY_METHOD_NAME,
DICT_FACTORY_METHOD_ARG_TYPES);
final Object dict = factoryMethod.invoke(null /* obj */,
new Object[] { context, locale, dictFile, dictNamePrefix, account });
return (ExpandableBinaryDictionary) dict;
} catch (final NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
Log.e(TAG, "Cannot create dictionary: " + dictType, e);
return null;
}
}
public void resetDictionaries(final Context context, final Locale[] newLocales,
final boolean useContactsDict, final boolean usePersonalizedDicts, final boolean useContactsDict, final boolean usePersonalizedDicts,
final boolean forceReloadMainDictionary, final boolean forceReloadMainDictionary,
@Nullable final String account, @Nullable final String account,
final DictionaryInitializationListener listener) { final DictionaryInitializationListener listener);
resetDictionariesWithDictNamePrefix(context, newLocales, useContactsDict,
usePersonalizedDicts, forceReloadMainDictionary, listener, "" /* dictNamePrefix */,
account);
}
@Nullable void resetDictionariesWithDictNamePrefix(final Context context,
static DictionaryGroup findDictionaryGroupWithLocale(final DictionaryGroup[] dictionaryGroups,
final Locale locale) {
for (DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (locale.equals(dictionaryGroup.mLocale)) {
return dictionaryGroup;
}
}
return null;
}
public void resetDictionariesWithDictNamePrefix(final Context context,
final Locale[] newLocales, final Locale[] newLocales,
final boolean useContactsDict, final boolean useContactsDict,
final boolean usePersonalizedDicts, final boolean usePersonalizedDicts,
final boolean forceReloadMainDictionary, final boolean forceReloadMainDictionary,
@Nullable final DictionaryInitializationListener listener, @Nullable final DictionaryInitializationListener listener,
final String dictNamePrefix, final String dictNamePrefix,
@Nullable final String account) { @Nullable final String account);
final HashMap<Locale, ArrayList<String>> existingDictionariesToCleanup = new HashMap<>();
// TODO: Make subDictTypesToUse configurable by resource or a static final list.
final HashSet<String> subDictTypesToUse = new HashSet<>();
subDictTypesToUse.add(Dictionary.TYPE_USER);
if (useContactsDict) {
subDictTypesToUse.add(Dictionary.TYPE_CONTACTS);
}
if (usePersonalizedDicts) {
subDictTypesToUse.add(Dictionary.TYPE_USER_HISTORY);
subDictTypesToUse.add(Dictionary.TYPE_PERSONALIZATION);
subDictTypesToUse.add(Dictionary.TYPE_CONTEXTUAL);
}
// Gather all dictionaries. We'll remove them from the list to clean up later.
for (final Locale newLocale : newLocales) {
final ArrayList<String> dictTypeForLocale = new ArrayList<>();
existingDictionariesToCleanup.put(newLocale, dictTypeForLocale);
final DictionaryGroup currentDictionaryGroupForLocale =
findDictionaryGroupWithLocale(mDictionaryGroups, newLocale);
if (null == currentDictionaryGroupForLocale) {
continue;
}
for (final String dictType : SUB_DICT_TYPES) {
if (currentDictionaryGroupForLocale.hasDict(dictType, account)) {
dictTypeForLocale.add(dictType);
}
}
if (currentDictionaryGroupForLocale.hasDict(Dictionary.TYPE_MAIN, account)) {
dictTypeForLocale.add(Dictionary.TYPE_MAIN);
}
}
final DictionaryGroup[] newDictionaryGroups = new DictionaryGroup[newLocales.length];
for (int i = 0; i < newLocales.length; ++i) {
final Locale newLocale = newLocales[i];
final DictionaryGroup dictionaryGroupForLocale =
findDictionaryGroupWithLocale(mDictionaryGroups, newLocale);
final ArrayList<String> dictTypesToCleanupForLocale =
existingDictionariesToCleanup.get(newLocale);
final boolean noExistingDictsForThisLocale = (null == dictionaryGroupForLocale);
final Dictionary mainDict;
if (forceReloadMainDictionary || noExistingDictsForThisLocale
|| !dictionaryGroupForLocale.hasDict(Dictionary.TYPE_MAIN, account)) {
mainDict = null;
} else {
mainDict = dictionaryGroupForLocale.getDict(Dictionary.TYPE_MAIN);
dictTypesToCleanupForLocale.remove(Dictionary.TYPE_MAIN);
}
final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>();
for (final String subDictType : subDictTypesToUse) {
final ExpandableBinaryDictionary subDict;
if (noExistingDictsForThisLocale
|| !dictionaryGroupForLocale.hasDict(subDictType, account)) {
// Create a new dictionary.
subDict = getSubDict(subDictType, context, newLocale, null /* dictFile */,
dictNamePrefix, account);
} else {
// Reuse the existing dictionary, and don't close it at the end
subDict = dictionaryGroupForLocale.getSubDict(subDictType);
dictTypesToCleanupForLocale.remove(subDictType);
}
subDicts.put(subDictType, subDict);
}
newDictionaryGroups[i] = new DictionaryGroup(newLocale, mainDict, account, subDicts);
}
// Replace Dictionaries.
final DictionaryGroup[] oldDictionaryGroups;
synchronized (mLock) {
oldDictionaryGroups = mDictionaryGroups;
mDictionaryGroups = newDictionaryGroups;
mMostProbableDictionaryGroup = newDictionaryGroups[0];
mIsUserDictEnabled = UserBinaryDictionary.isEnabled(context);
if (hasAtLeastOneUninitializedMainDictionary()) {
asyncReloadUninitializedMainDictionaries(context, newLocales, listener);
}
}
if (listener != null) {
listener.onUpdateMainDictionaryAvailability(hasAtLeastOneInitializedMainDictionary());
}
// Clean up old dictionaries.
for (final Locale localeToCleanUp : existingDictionariesToCleanup.keySet()) {
final ArrayList<String> dictTypesToCleanUp =
existingDictionariesToCleanup.get(localeToCleanUp);
final DictionaryGroup dictionarySetToCleanup =
findDictionaryGroupWithLocale(oldDictionaryGroups, localeToCleanUp);
for (final String dictType : dictTypesToCleanUp) {
dictionarySetToCleanup.closeDict(dictType);
}
}
}
private void asyncReloadUninitializedMainDictionaries(final Context context,
final Locale[] locales, final DictionaryInitializationListener listener) {
final CountDownLatch latchForWaitingLoadingMainDictionary = new CountDownLatch(1);
mLatchForWaitingLoadingMainDictionaries = latchForWaitingLoadingMainDictionary;
ExecutorUtils.getExecutor("InitializeBinaryDictionary").execute(new Runnable() {
@Override
public void run() {
doReloadUninitializedMainDictionaries(
context, locales, listener, latchForWaitingLoadingMainDictionary);
}
});
}
void doReloadUninitializedMainDictionaries(final Context context, final Locale[] locales,
final DictionaryInitializationListener listener,
final CountDownLatch latchForWaitingLoadingMainDictionary) {
for (final Locale locale : locales) {
final DictionaryGroup dictionaryGroup =
findDictionaryGroupWithLocale(mDictionaryGroups, locale);
if (null == dictionaryGroup) {
// This should never happen, but better safe than crashy
Log.w(TAG, "Expected a dictionary group for " + locale + " but none found");
continue;
}
final Dictionary mainDict =
DictionaryFactory.createMainDictionaryFromManager(context, locale);
synchronized (mLock) {
if (locale.equals(dictionaryGroup.mLocale)) {
dictionaryGroup.setMainDict(mainDict);
} else {
// Dictionary facilitator has been reset for another locale.
mainDict.close();
}
}
}
if (listener != null) {
listener.onUpdateMainDictionaryAvailability(
hasAtLeastOneInitializedMainDictionary());
}
latchForWaitingLoadingMainDictionary.countDown();
}
@UsedForTesting @UsedForTesting
public void resetDictionariesForTesting(final Context context, final Locale[] locales, void resetDictionariesForTesting(final Context context, final Locale[] locales,
final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles, final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles,
final Map<String, Map<String, String>> additionalDictAttributes, final Map<String, Map<String, String>> additionalDictAttributes,
@Nullable final String account) { @Nullable final String account);
Dictionary mainDictionary = null;
final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>();
final DictionaryGroup[] dictionaryGroups = new DictionaryGroup[locales.length]; void closeDictionaries();
for (int i = 0; i < locales.length; ++i) {
final Locale locale = locales[i];
for (final String dictType : dictionaryTypes) {
if (dictType.equals(Dictionary.TYPE_MAIN)) {
mainDictionary = DictionaryFactory.createMainDictionaryFromManager(context,
locale);
} else {
final File dictFile = dictionaryFiles.get(dictType);
final ExpandableBinaryDictionary dict = getSubDict(
dictType, context, locale, dictFile, "" /* dictNamePrefix */, account);
if (additionalDictAttributes.containsKey(dictType)) {
dict.clearAndFlushDictionaryWithAdditionalAttributes(
additionalDictAttributes.get(dictType));
}
if (dict == null) {
throw new RuntimeException("Unknown dictionary type: " + dictType);
}
dict.reloadDictionaryIfRequired();
dict.waitAllTasksForTests();
subDicts.put(dictType, dict);
}
}
dictionaryGroups[i] = new DictionaryGroup(locale, mainDictionary, account, subDicts);
}
mDictionaryGroups = dictionaryGroups;
mMostProbableDictionaryGroup = dictionaryGroups[0];
}
public void closeDictionaries() {
final DictionaryGroup[] dictionaryGroups;
synchronized (mLock) {
dictionaryGroups = mDictionaryGroups;
mMostProbableDictionaryGroup = new DictionaryGroup();
mDictionaryGroups = new DictionaryGroup[] { mMostProbableDictionaryGroup };
}
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
dictionaryGroup.closeDict(dictType);
}
}
mDistracterFilter.close();
if (mPersonalizationHelper != null) {
mPersonalizationHelper.close();
}
}
@UsedForTesting @UsedForTesting
public ExpandableBinaryDictionary getSubDictForTesting(final String dictName) { ExpandableBinaryDictionary getSubDictForTesting(final String dictName);
return mMostProbableDictionaryGroup.getSubDict(dictName);
}
// The main dictionaries are loaded asynchronously. Don't cache the return value // The main dictionaries are loaded asynchronously. Don't cache the return value
// of these methods. // of these methods.
public boolean hasAtLeastOneInitializedMainDictionary() { boolean hasAtLeastOneInitializedMainDictionary();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN);
if (mainDict != null && mainDict.isInitialized()) {
return true;
}
}
return false;
}
public boolean hasAtLeastOneUninitializedMainDictionary() { boolean hasAtLeastOneUninitializedMainDictionary();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN);
if (mainDict == null || !mainDict.isInitialized()) {
return true;
}
}
return false;
}
public boolean hasPersonalizationDictionary() { boolean hasPersonalizationDictionary();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (dictionaryGroup.hasDict(Dictionary.TYPE_PERSONALIZATION, null /* account */)) {
return true;
}
}
return false;
}
public void flushPersonalizationDictionary() { void flushPersonalizationDictionary();
final HashSet<ExpandableBinaryDictionary> personalizationDictsUsedForSuggestion =
new HashSet<>();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary personalizationDictUsedForSuggestion =
dictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
personalizationDictsUsedForSuggestion.add(personalizationDictUsedForSuggestion);
}
mPersonalizationHelper.flushPersonalizationDictionariesToUpdate(
personalizationDictsUsedForSuggestion);
mDistracterFilter.close();
}
public void waitForLoadingMainDictionaries(final long timeout, final TimeUnit unit) void waitForLoadingMainDictionaries(final long timeout, final TimeUnit unit)
throws InterruptedException { throws InterruptedException;
mLatchForWaitingLoadingMainDictionaries.await(timeout, unit);
}
@UsedForTesting @UsedForTesting
public void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit) void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit)
throws InterruptedException { throws InterruptedException;
waitForLoadingMainDictionaries(timeout, unit);
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final ExpandableBinaryDictionary dict : dictionaryGroup.mSubDictMap.values()) {
dict.waitAllTasksForTests();
}
}
}
public boolean isUserDictionaryEnabled() { boolean isUserDictionaryEnabled();
return mIsUserDictEnabled;
}
public void addWordToUserDictionary(final Context context, final String word) { void addWordToUserDictionary(final Context context, final String word);
final Locale locale = getMostProbableLocale();
if (locale == null) {
return;
}
// TODO: add a toast telling what language this is being added to?
UserBinaryDictionary.addWordToUserDictionary(context, locale, word);
}
public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized,
@Nonnull final NgramContext ngramContext, final int timeStampInSeconds, @Nonnull final NgramContext ngramContext, final int timeStampInSeconds,
final boolean blockPotentiallyOffensive) { final boolean blockPotentiallyOffensive);
final DictionaryGroup dictionaryGroup = getDictionaryGroupForMostProbableLanguage();
final String[] words = suggestion.split(Constants.WORD_SEPARATOR);
NgramContext ngramContextForCurrentWord = ngramContext;
for (int i = 0; i < words.length; i++) {
final String currentWord = words[i];
final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false;
addWordToUserHistory(dictionaryGroup, ngramContextForCurrentWord, currentWord,
wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive);
ngramContextForCurrentWord =
ngramContextForCurrentWord.getNextNgramContext(new WordInfo(currentWord));
}
}
private void addWordToUserHistory(final DictionaryGroup dictionaryGroup, void removeWordFromPersonalizedDicts(final String word);
final NgramContext ngramContext, final String word, final boolean wasAutoCapitalized,
final int timeStampInSeconds, final boolean blockPotentiallyOffensive) {
final ExpandableBinaryDictionary userHistoryDictionary =
dictionaryGroup.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary == null
|| !isConfidentAboutCurrentLanguageBeing(userHistoryDictionary.mLocale)) {
return;
}
final int maxFreq = getFrequency(word);
if (maxFreq == 0 && blockPotentiallyOffensive) {
return;
}
final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale);
final String secondWord;
if (wasAutoCapitalized) {
if (isValidWord(word, false /* ignoreCase */)
&& !isValidWord(lowerCasedWord, false /* ignoreCase */)) {
// If the word was auto-capitalized and exists only as a capitalized word in the
// dictionary, then we must not downcase it before registering it. For example,
// the name of the contacts in start-of-sentence position would come here with the
// wasAutoCapitalized flag: if we downcase it, we'd register a lower-case version
// of that contact's name which would end up popping in suggestions.
secondWord = word;
} else {
// If however the word is not in the dictionary, or exists as a lower-case word
// only, then we consider that was a lower-case word that had been auto-capitalized.
secondWord = lowerCasedWord;
}
} else {
// HACK: We'd like to avoid adding the capitalized form of common words to the User
// History dictionary in order to avoid suggesting them until the dictionary
// consolidation is done.
// TODO: Remove this hack when ready.
final int lowerCaseFreqInMainDict = dictionaryGroup.hasDict(Dictionary.TYPE_MAIN,
null /* account */) ?
dictionaryGroup.getDict(Dictionary.TYPE_MAIN).getFrequency(lowerCasedWord) :
Dictionary.NOT_A_PROBABILITY;
if (maxFreq < lowerCaseFreqInMainDict
&& lowerCaseFreqInMainDict >= CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT) {
// Use lower cased word as the word can be a distracter of the popular word.
secondWord = lowerCasedWord;
} else {
secondWord = word;
}
}
// We demote unrecognized words (frequency < 0, below) by specifying them as "invalid".
// We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, ngramContext, secondWord,
isValid, timeStampInSeconds,
new DistracterFilterCheckingIsInDictionary(
mDistracterFilter, userHistoryDictionary));
}
private void removeWord(final String dictName, final String word) {
final ExpandableBinaryDictionary dictionary =
getDictionaryGroupForMostProbableLanguage().getSubDict(dictName);
if (dictionary != null) {
dictionary.removeUnigramEntryDynamically(word);
}
}
public void removeWordFromPersonalizedDicts(final String word) {
removeWord(Dictionary.TYPE_USER_HISTORY, word);
removeWord(Dictionary.TYPE_PERSONALIZATION, word);
removeWord(Dictionary.TYPE_CONTEXTUAL, word);
}
// TODO: Revise the way to fusion suggestion results. // TODO: Revise the way to fusion suggestion results.
public SuggestionResults getSuggestionResults(final WordComposer composer, SuggestionResults getSuggestionResults(final WordComposer composer,
final NgramContext ngramContext, final long proximityInfoHandle, final NgramContext ngramContext, final long proximityInfoHandle,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId) { final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId);
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
final SuggestionResults suggestionResults = new SuggestionResults(
SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext());
final float[] weightOfLangModelVsSpatialModel =
new float[] { Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL };
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
if (null == dictionary) continue;
final float weightForLocale = composer.isBatchMode()
? dictionaryGroup.mWeightForGesturingInLocale
: dictionaryGroup.mWeightForTypingInLocale;
final ArrayList<SuggestedWordInfo> dictionarySuggestions =
dictionary.getSuggestions(composer.getComposedDataSnapshot(), ngramContext,
proximityInfoHandle, settingsValuesForSuggestion, sessionId,
weightForLocale, weightOfLangModelVsSpatialModel);
if (null == dictionarySuggestions) continue;
suggestionResults.addAll(dictionarySuggestions);
if (null != suggestionResults.mRawSuggestions) {
suggestionResults.mRawSuggestions.addAll(dictionarySuggestions);
}
}
}
return suggestionResults;
}
public boolean isValidWord(final String word, final boolean ignoreCase) { boolean isValidWord(final String word, final boolean ignoreCase);
if (TextUtils.isEmpty(word)) {
return false;
}
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (dictionaryGroup.mLocale == null) {
continue;
}
final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale);
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
// Ideally the passed map would come out of a {@link java.util.concurrent.Future} and
// would be immutable once it's finished initializing, but concretely a null test is
// probably good enough for the time being.
if (null == dictionary) continue;
if (dictionary.isValidWord(word)
|| (ignoreCase && dictionary.isValidWord(lowerCasedWord))) {
return true;
}
}
}
return false;
}
private int getFrequencyInternal(final String word, int getFrequency(final String word);
final boolean isGettingMaxFrequencyOfExactMatches) {
if (TextUtils.isEmpty(word)) {
return Dictionary.NOT_A_PROBABILITY;
}
int maxFreq = Dictionary.NOT_A_PROBABILITY;
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
if (dictionary == null) continue;
final int tempFreq;
if (isGettingMaxFrequencyOfExactMatches) {
tempFreq = dictionary.getMaxFrequencyOfExactMatches(word);
} else {
tempFreq = dictionary.getFrequency(word);
}
if (tempFreq >= maxFreq) {
maxFreq = tempFreq;
}
}
}
return maxFreq;
}
public int getFrequency(final String word) { int getMaxFrequencyOfExactMatches(final String word);
return getFrequencyInternal(word, false /* isGettingMaxFrequencyOfExactMatches */);
}
public int getMaxFrequencyOfExactMatches(final String word) { void clearUserHistoryDictionary();
return getFrequencyInternal(word, true /* isGettingMaxFrequencyOfExactMatches */);
}
private void clearSubDictionary(final String dictName) {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictName);
if (dictionary != null) {
dictionary.clear();
}
}
}
public void clearUserHistoryDictionary() {
clearSubDictionary(Dictionary.TYPE_USER_HISTORY);
}
// This method gets called only when the IME receives a notification to remove the // This method gets called only when the IME receives a notification to remove the
// personalization dictionary. // personalization dictionary.
public void clearPersonalizationDictionary() { void clearPersonalizationDictionary();
clearSubDictionary(Dictionary.TYPE_PERSONALIZATION);
mPersonalizationHelper.clearDictionariesToUpdate();
}
public void clearContextualDictionary() { void clearContextualDictionary();
clearSubDictionary(Dictionary.TYPE_CONTEXTUAL);
}
public void addEntriesToPersonalizationDictionary( void addEntriesToPersonalizationDictionary(
final PersonalizationDataChunk personalizationDataChunk, final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations, final SpacingAndPunctuations spacingAndPunctuations,
final UpdateEntriesForInputEventsCallback callback) { final UpdateEntriesForInputEventsCallback callback);
mPersonalizationHelper.updateEntriesOfPersonalizationDictionaries(
getMostProbableLocale(), personalizationDataChunk, spacingAndPunctuations,
callback);
}
@UsedForTesting @UsedForTesting
public void addPhraseToContextualDictionary(final String[] phrase, final int probability, void addPhraseToContextualDictionary(final String[] phrase, final int probability,
final int bigramProbabilityForWords, final int bigramProbabilityForPhrases) { final int bigramProbabilityForWords, final int bigramProbabilityForPhrases);
// TODO: we're inserting the phrase into the dictionary for the active language. Rethink
// this a bit from a theoretical point of view.
final ExpandableBinaryDictionary contextualDict =
getDictionaryGroupForMostProbableLanguage().getSubDict(Dictionary.TYPE_CONTEXTUAL);
if (contextualDict == null) {
return;
}
NgramContext ngramContext = NgramContext.BEGINNING_OF_SENTENCE;
for (int i = 0; i < phrase.length; i++) {
final String[] subPhrase = Arrays.copyOfRange(phrase, i /* start */, phrase.length);
final String subPhraseStr = TextUtils.join(Constants.WORD_SEPARATOR, subPhrase);
contextualDict.addUnigramEntryWithCheckingDistracter(
subPhraseStr, probability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY /* shortcutFreq */,
false /* isNotAWord */, false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
contextualDict.addNgramEntry(ngramContext, subPhraseStr,
bigramProbabilityForPhrases, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (i < phrase.length - 1) { void dumpDictionaryForDebug(final String dictName);
contextualDict.addUnigramEntryWithCheckingDistracter(
phrase[i], probability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY /* shortcutFreq */,
false /* isNotAWord */, false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
contextualDict.addNgramEntry(ngramContext, phrase[i],
bigramProbabilityForWords, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
}
ngramContext =
ngramContext.getNextNgramContext(new NgramContext.WordInfo(phrase[i]));
}
}
public void dumpDictionaryForDebug(final String dictName) { ArrayList<Pair<String, DictionaryStats>> getStatsOfEnabledSubDicts();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary dictToDump = dictionaryGroup.getSubDict(dictName);
if (dictToDump == null) {
Log.e(TAG, "Cannot dump " + dictName + ". "
+ "The dictionary is not being used for suggestion or cannot be dumped.");
return;
}
dictToDump.dumpAllWordsForDebug();
}
}
public ArrayList<Pair<String, DictionaryStats>> getStatsOfEnabledSubDicts() {
final ArrayList<Pair<String, DictionaryStats>> statsOfEnabledSubDicts = new ArrayList<>();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : SUB_DICT_TYPES) {
final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictType);
if (dictionary == null) continue;
statsOfEnabledSubDicts.add(new Pair<>(dictType, dictionary.getDictionaryStats()));
}
}
return statsOfEnabledSubDicts;
}
} }

947
java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java Normal file
View File

@ -0,0 +1,947 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
import android.content.Context;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.UpdateEntriesForInputEventsCallback;
import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.common.Constants;
import com.android.inputmethod.latin.personalization.ContextualDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationDataChunk;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.SuggestionResults;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/**
* Facilitates interaction with different kinds of dictionaries. Provides APIs
* to instantiate and select the correct dictionaries (based on language or account),
* update entries and fetch suggestions.
*
* Currently AndroidSpellCheckerService and LatinIME both use DictionaryFacilitator as
* a client for interacting with dictionaries.
*/
public class DictionaryFacilitatorImpl implements DictionaryFacilitator {
// TODO: Consolidate dictionaries in native code.
public static final String TAG = DictionaryFacilitatorImpl.class.getSimpleName();
// HACK: This threshold is being used when adding a capitalized entry in the User History
// dictionary.
private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140;
// How many words we need to type in a row ({@see mConfidenceInMostProbableLanguage}) to
// declare we are confident the user is typing in the most probable language.
private static final int CONFIDENCE_THRESHOLD = 3;
private DictionaryGroup[] mDictionaryGroups = new DictionaryGroup[] { new DictionaryGroup() };
private DictionaryGroup mMostProbableDictionaryGroup = mDictionaryGroups[0];
private boolean mIsUserDictEnabled = false;
private volatile CountDownLatch mLatchForWaitingLoadingMainDictionaries = new CountDownLatch(0);
// To synchronize assigning mDictionaryGroup to ensure closing dictionaries.
private final Object mLock = new Object();
private final DistracterFilter mDistracterFilter;
private final PersonalizationHelperForDictionaryFacilitator mPersonalizationHelper;
private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
new String[] {
Dictionary.TYPE_MAIN,
Dictionary.TYPE_USER_HISTORY,
Dictionary.TYPE_PERSONALIZATION,
Dictionary.TYPE_USER,
Dictionary.TYPE_CONTACTS,
Dictionary.TYPE_CONTEXTUAL
};
public static final Map<String, Class<? extends ExpandableBinaryDictionary>>
DICT_TYPE_TO_CLASS = new HashMap<>();
static {
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER_HISTORY, UserHistoryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_PERSONALIZATION, PersonalizationDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER, UserBinaryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTACTS, ContactsBinaryDictionary.class);
DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTEXTUAL, ContextualDictionary.class);
}
private static final String DICT_FACTORY_METHOD_NAME = "getDictionary";
private static final Class<?>[] DICT_FACTORY_METHOD_ARG_TYPES =
new Class[] { Context.class, Locale.class, File.class, String.class, String.class };
private static final String[] SUB_DICT_TYPES =
Arrays.copyOfRange(DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS, 1 /* start */,
DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS.length);
/**
* Returns whether this facilitator is exactly for this list of locales.
*
* @param locales the list of locales to test against
*/
public boolean isForLocales(final Locale[] locales) {
if (locales.length != mDictionaryGroups.length) {
return false;
}
for (final Locale locale : locales) {
boolean found = false;
for (final DictionaryGroup group : mDictionaryGroups) {
if (locale.equals(group.mLocale)) {
found = true;
break;
}
}
if (!found) {
return false;
}
}
return true;
}
/**
* Returns whether this facilitator is exactly for this account.
*
* @param account the account to test against.
*/
public boolean isForAccount(@Nullable final String account) {
for (final DictionaryGroup group : mDictionaryGroups) {
if (!TextUtils.equals(group.mAccount, account)) {
return false;
}
}
return true;
}
/**
* A group of dictionaries that work together for a single language.
*/
private static class DictionaryGroup {
// TODO: Add null analysis annotations.
// TODO: Run evaluation to determine a reasonable value for these constants. The current
// values are ad-hoc and chosen without any particular care or methodology.
public static final float WEIGHT_FOR_MOST_PROBABLE_LANGUAGE = 1.0f;
public static final float WEIGHT_FOR_GESTURING_IN_NOT_MOST_PROBABLE_LANGUAGE = 0.95f;
public static final float WEIGHT_FOR_TYPING_IN_NOT_MOST_PROBABLE_LANGUAGE = 0.6f;
/**
* The locale associated with the dictionary group.
*/
@Nullable public final Locale mLocale;
/**
* The user account associated with the dictionary group.
*/
@Nullable public final String mAccount;
@Nullable private Dictionary mMainDict;
// Confidence that the most probable language is actually the language the user is
// typing in. For now, this is simply the number of times a word from this language
// has been committed in a row.
private int mConfidence = 0;
public float mWeightForTypingInLocale = WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
public float mWeightForGesturingInLocale = WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap =
new ConcurrentHashMap<>();
public DictionaryGroup() {
this(null /* locale */, null /* mainDict */, null /* account */,
Collections.<String, ExpandableBinaryDictionary>emptyMap() /* subDicts */);
}
public DictionaryGroup(@Nullable final Locale locale,
@Nullable final Dictionary mainDict,
@Nullable final String account,
final Map<String, ExpandableBinaryDictionary> subDicts) {
mLocale = locale;
mAccount = account;
// The main dictionary can be asynchronously loaded.
setMainDict(mainDict);
for (final Map.Entry<String, ExpandableBinaryDictionary> entry : subDicts.entrySet()) {
setSubDict(entry.getKey(), entry.getValue());
}
}
private void setSubDict(final String dictType, final ExpandableBinaryDictionary dict) {
if (dict != null) {
mSubDictMap.put(dictType, dict);
}
}
public void setMainDict(final Dictionary mainDict) {
// Close old dictionary if exists. Main dictionary can be assigned multiple times.
final Dictionary oldDict = mMainDict;
mMainDict = mainDict;
if (oldDict != null && mainDict != oldDict) {
oldDict.close();
}
}
public Dictionary getDict(final String dictType) {
if (Dictionary.TYPE_MAIN.equals(dictType)) {
return mMainDict;
}
return getSubDict(dictType);
}
public ExpandableBinaryDictionary getSubDict(final String dictType) {
return mSubDictMap.get(dictType);
}
public boolean hasDict(final String dictType, @Nullable final String account) {
if (Dictionary.TYPE_MAIN.equals(dictType)) {
return mMainDict != null;
}
if (Dictionary.TYPE_USER_HISTORY.equals(dictType) &&
!TextUtils.equals(account, mAccount)) {
// If the dictionary type is user history, & if the account doesn't match,
// return immediately. If the account matches, continue looking it up in the
// sub dictionary map.
return false;
}
return mSubDictMap.containsKey(dictType);
}
public void closeDict(final String dictType) {
final Dictionary dict;
if (Dictionary.TYPE_MAIN.equals(dictType)) {
dict = mMainDict;
} else {
dict = mSubDictMap.remove(dictType);
}
if (dict != null) {
dict.close();
}
}
}
public DictionaryFacilitatorImpl() {
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
mPersonalizationHelper = null;
}
public DictionaryFacilitatorImpl(final Context context) {
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);
mPersonalizationHelper =
new PersonalizationHelperForDictionaryFacilitator(context, mDistracterFilter);
}
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
mDistracterFilter.updateEnabledSubtypes(enabledSubtypes);
mPersonalizationHelper.updateEnabledSubtypes(enabledSubtypes);
}
// TODO: remove this, it's confusing with seamless multiple language switching
public void setIsMonolingualUser(final boolean isMonolingualUser) {
mPersonalizationHelper.setIsMonolingualUser(isMonolingualUser);
}
public boolean isActive() {
return null != mDictionaryGroups[0].mLocale;
}
/**
* Returns the most probable locale among all currently active locales. BE CAREFUL using this.
*
* DO NOT USE THIS just because it's convenient. Use it when it's correct, for example when
* choosing what dictionary to put a word in, or when changing the capitalization of a typed
* string.
* @return the most probable locale
*/
public Locale getMostProbableLocale() {
return getDictionaryGroupForMostProbableLanguage().mLocale;
}
public Locale[] getLocales() {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
final Locale[] locales = new Locale[dictionaryGroups.length];
for (int i = 0; i < dictionaryGroups.length; ++i) {
locales[i] = dictionaryGroups[i].mLocale;
}
return locales;
}
private DictionaryGroup getDictionaryGroupForMostProbableLanguage() {
return mMostProbableDictionaryGroup;
}
public void switchMostProbableLanguage(@Nullable final Locale locale) {
if (null == locale) {
// In many cases, there is no locale to a committed word. For example, a typed word
// that is in none of the currently active dictionaries but still does not
// auto-correct to anything has no locale. In this case we simply do not change
// the most probable language and do not touch confidence.
return;
}
final DictionaryGroup newMostProbableDictionaryGroup =
findDictionaryGroupWithLocale(mDictionaryGroups, locale);
if (null == newMostProbableDictionaryGroup) {
// It seems this may happen as a race condition; pressing the globe key and space
// in quick succession could commit a word out of a dictionary that's not in the
// facilitator any more. In this case, just not changing things is fine.
return;
}
if (newMostProbableDictionaryGroup == mMostProbableDictionaryGroup) {
++newMostProbableDictionaryGroup.mConfidence;
} else {
mMostProbableDictionaryGroup.mWeightForTypingInLocale =
DictionaryGroup.WEIGHT_FOR_TYPING_IN_NOT_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup.mWeightForGesturingInLocale =
DictionaryGroup.WEIGHT_FOR_GESTURING_IN_NOT_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup.mConfidence = 0;
newMostProbableDictionaryGroup.mWeightForTypingInLocale =
DictionaryGroup.WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
newMostProbableDictionaryGroup.mWeightForGesturingInLocale =
DictionaryGroup.WEIGHT_FOR_MOST_PROBABLE_LANGUAGE;
mMostProbableDictionaryGroup = newMostProbableDictionaryGroup;
}
}
public boolean isConfidentAboutCurrentLanguageBeing(final Locale mLocale) {
final DictionaryGroup mostProbableDictionaryGroup = mMostProbableDictionaryGroup;
if (!mostProbableDictionaryGroup.mLocale.equals(mLocale)) {
return false;
}
if (mDictionaryGroups.length <= 1) {
return true;
}
return mostProbableDictionaryGroup.mConfidence >= CONFIDENCE_THRESHOLD;
}
@Nullable
private static ExpandableBinaryDictionary getSubDict(final String dictType,
final Context context, final Locale locale, final File dictFile,
final String dictNamePrefix, @Nullable final String account) {
final Class<? extends ExpandableBinaryDictionary> dictClass =
DICT_TYPE_TO_CLASS.get(dictType);
if (dictClass == null) {
return null;
}
try {
final Method factoryMethod = dictClass.getMethod(DICT_FACTORY_METHOD_NAME,
DICT_FACTORY_METHOD_ARG_TYPES);
final Object dict = factoryMethod.invoke(null /* obj */,
new Object[] { context, locale, dictFile, dictNamePrefix, account });
return (ExpandableBinaryDictionary) dict;
} catch (final NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
Log.e(TAG, "Cannot create dictionary: " + dictType, e);
return null;
}
}
public void resetDictionaries(final Context context, final Locale[] newLocales,
final boolean useContactsDict, final boolean usePersonalizedDicts,
final boolean forceReloadMainDictionary,
@Nullable final String account,
final DictionaryInitializationListener listener) {
resetDictionariesWithDictNamePrefix(context, newLocales, useContactsDict,
usePersonalizedDicts, forceReloadMainDictionary, listener, "" /* dictNamePrefix */,
account);
}
@Nullable
static DictionaryGroup findDictionaryGroupWithLocale(final DictionaryGroup[] dictionaryGroups,
final Locale locale) {
for (DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (locale.equals(dictionaryGroup.mLocale)) {
return dictionaryGroup;
}
}
return null;
}
public void resetDictionariesWithDictNamePrefix(final Context context,
final Locale[] newLocales,
final boolean useContactsDict,
final boolean usePersonalizedDicts,
final boolean forceReloadMainDictionary,
@Nullable final DictionaryInitializationListener listener,
final String dictNamePrefix,
@Nullable final String account) {
final HashMap<Locale, ArrayList<String>> existingDictionariesToCleanup = new HashMap<>();
// TODO: Make subDictTypesToUse configurable by resource or a static final list.
final HashSet<String> subDictTypesToUse = new HashSet<>();
subDictTypesToUse.add(Dictionary.TYPE_USER);
if (useContactsDict) {
subDictTypesToUse.add(Dictionary.TYPE_CONTACTS);
}
if (usePersonalizedDicts) {
subDictTypesToUse.add(Dictionary.TYPE_USER_HISTORY);
subDictTypesToUse.add(Dictionary.TYPE_PERSONALIZATION);
subDictTypesToUse.add(Dictionary.TYPE_CONTEXTUAL);
}
// Gather all dictionaries. We'll remove them from the list to clean up later.
for (final Locale newLocale : newLocales) {
final ArrayList<String> dictTypeForLocale = new ArrayList<>();
existingDictionariesToCleanup.put(newLocale, dictTypeForLocale);
final DictionaryGroup currentDictionaryGroupForLocale =
findDictionaryGroupWithLocale(mDictionaryGroups, newLocale);
if (null == currentDictionaryGroupForLocale) {
continue;
}
for (final String dictType : SUB_DICT_TYPES) {
if (currentDictionaryGroupForLocale.hasDict(dictType, account)) {
dictTypeForLocale.add(dictType);
}
}
if (currentDictionaryGroupForLocale.hasDict(Dictionary.TYPE_MAIN, account)) {
dictTypeForLocale.add(Dictionary.TYPE_MAIN);
}
}
final DictionaryGroup[] newDictionaryGroups = new DictionaryGroup[newLocales.length];
for (int i = 0; i < newLocales.length; ++i) {
final Locale newLocale = newLocales[i];
final DictionaryGroup dictionaryGroupForLocale =
findDictionaryGroupWithLocale(mDictionaryGroups, newLocale);
final ArrayList<String> dictTypesToCleanupForLocale =
existingDictionariesToCleanup.get(newLocale);
final boolean noExistingDictsForThisLocale = (null == dictionaryGroupForLocale);
final Dictionary mainDict;
if (forceReloadMainDictionary || noExistingDictsForThisLocale
|| !dictionaryGroupForLocale.hasDict(Dictionary.TYPE_MAIN, account)) {
mainDict = null;
} else {
mainDict = dictionaryGroupForLocale.getDict(Dictionary.TYPE_MAIN);
dictTypesToCleanupForLocale.remove(Dictionary.TYPE_MAIN);
}
final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>();
for (final String subDictType : subDictTypesToUse) {
final ExpandableBinaryDictionary subDict;
if (noExistingDictsForThisLocale
|| !dictionaryGroupForLocale.hasDict(subDictType, account)) {
// Create a new dictionary.
subDict = getSubDict(subDictType, context, newLocale, null /* dictFile */,
dictNamePrefix, account);
} else {
// Reuse the existing dictionary, and don't close it at the end
subDict = dictionaryGroupForLocale.getSubDict(subDictType);
dictTypesToCleanupForLocale.remove(subDictType);
}
subDicts.put(subDictType, subDict);
}
newDictionaryGroups[i] = new DictionaryGroup(newLocale, mainDict, account, subDicts);
}
// Replace Dictionaries.
final DictionaryGroup[] oldDictionaryGroups;
synchronized (mLock) {
oldDictionaryGroups = mDictionaryGroups;
mDictionaryGroups = newDictionaryGroups;
mMostProbableDictionaryGroup = newDictionaryGroups[0];
mIsUserDictEnabled = UserBinaryDictionary.isEnabled(context);
if (hasAtLeastOneUninitializedMainDictionary()) {
asyncReloadUninitializedMainDictionaries(context, newLocales, listener);
}
}
if (listener != null) {
listener.onUpdateMainDictionaryAvailability(hasAtLeastOneInitializedMainDictionary());
}
// Clean up old dictionaries.
for (final Locale localeToCleanUp : existingDictionariesToCleanup.keySet()) {
final ArrayList<String> dictTypesToCleanUp =
existingDictionariesToCleanup.get(localeToCleanUp);
final DictionaryGroup dictionarySetToCleanup =
findDictionaryGroupWithLocale(oldDictionaryGroups, localeToCleanUp);
for (final String dictType : dictTypesToCleanUp) {
dictionarySetToCleanup.closeDict(dictType);
}
}
}
private void asyncReloadUninitializedMainDictionaries(final Context context,
final Locale[] locales, final DictionaryInitializationListener listener) {
final CountDownLatch latchForWaitingLoadingMainDictionary = new CountDownLatch(1);
mLatchForWaitingLoadingMainDictionaries = latchForWaitingLoadingMainDictionary;
ExecutorUtils.getExecutor("InitializeBinaryDictionary").execute(new Runnable() {
@Override
public void run() {
doReloadUninitializedMainDictionaries(
context, locales, listener, latchForWaitingLoadingMainDictionary);
}
});
}
void doReloadUninitializedMainDictionaries(final Context context, final Locale[] locales,
final DictionaryInitializationListener listener,
final CountDownLatch latchForWaitingLoadingMainDictionary) {
for (final Locale locale : locales) {
final DictionaryGroup dictionaryGroup =
findDictionaryGroupWithLocale(mDictionaryGroups, locale);
if (null == dictionaryGroup) {
// This should never happen, but better safe than crashy
Log.w(TAG, "Expected a dictionary group for " + locale + " but none found");
continue;
}
final Dictionary mainDict =
DictionaryFactory.createMainDictionaryFromManager(context, locale);
synchronized (mLock) {
if (locale.equals(dictionaryGroup.mLocale)) {
dictionaryGroup.setMainDict(mainDict);
} else {
// Dictionary facilitator has been reset for another locale.
mainDict.close();
}
}
}
if (listener != null) {
listener.onUpdateMainDictionaryAvailability(
hasAtLeastOneInitializedMainDictionary());
}
latchForWaitingLoadingMainDictionary.countDown();
}
@UsedForTesting
public void resetDictionariesForTesting(final Context context, final Locale[] locales,
final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles,
final Map<String, Map<String, String>> additionalDictAttributes,
@Nullable final String account) {
Dictionary mainDictionary = null;
final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>();
final DictionaryGroup[] dictionaryGroups = new DictionaryGroup[locales.length];
for (int i = 0; i < locales.length; ++i) {
final Locale locale = locales[i];
for (final String dictType : dictionaryTypes) {
if (dictType.equals(Dictionary.TYPE_MAIN)) {
mainDictionary = DictionaryFactory.createMainDictionaryFromManager(context,
locale);
} else {
final File dictFile = dictionaryFiles.get(dictType);
final ExpandableBinaryDictionary dict = getSubDict(
dictType, context, locale, dictFile, "" /* dictNamePrefix */, account);
if (additionalDictAttributes.containsKey(dictType)) {
dict.clearAndFlushDictionaryWithAdditionalAttributes(
additionalDictAttributes.get(dictType));
}
if (dict == null) {
throw new RuntimeException("Unknown dictionary type: " + dictType);
}
dict.reloadDictionaryIfRequired();
dict.waitAllTasksForTests();
subDicts.put(dictType, dict);
}
}
dictionaryGroups[i] = new DictionaryGroup(locale, mainDictionary, account, subDicts);
}
mDictionaryGroups = dictionaryGroups;
mMostProbableDictionaryGroup = dictionaryGroups[0];
}
public void closeDictionaries() {
final DictionaryGroup[] dictionaryGroups;
synchronized (mLock) {
dictionaryGroups = mDictionaryGroups;
mMostProbableDictionaryGroup = new DictionaryGroup();
mDictionaryGroups = new DictionaryGroup[] { mMostProbableDictionaryGroup };
}
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
dictionaryGroup.closeDict(dictType);
}
}
mDistracterFilter.close();
if (mPersonalizationHelper != null) {
mPersonalizationHelper.close();
}
}
@UsedForTesting
public ExpandableBinaryDictionary getSubDictForTesting(final String dictName) {
return mMostProbableDictionaryGroup.getSubDict(dictName);
}
// The main dictionaries are loaded asynchronously. Don't cache the return value
// of these methods.
public boolean hasAtLeastOneInitializedMainDictionary() {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN);
if (mainDict != null && mainDict.isInitialized()) {
return true;
}
}
return false;
}
public boolean hasAtLeastOneUninitializedMainDictionary() {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN);
if (mainDict == null || !mainDict.isInitialized()) {
return true;
}
}
return false;
}
public boolean hasPersonalizationDictionary() {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (dictionaryGroup.hasDict(Dictionary.TYPE_PERSONALIZATION, null /* account */)) {
return true;
}
}
return false;
}
public void flushPersonalizationDictionary() {
final HashSet<ExpandableBinaryDictionary> personalizationDictsUsedForSuggestion =
new HashSet<>();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary personalizationDictUsedForSuggestion =
dictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
personalizationDictsUsedForSuggestion.add(personalizationDictUsedForSuggestion);
}
mPersonalizationHelper.flushPersonalizationDictionariesToUpdate(
personalizationDictsUsedForSuggestion);
mDistracterFilter.close();
}
public void waitForLoadingMainDictionaries(final long timeout, final TimeUnit unit)
throws InterruptedException {
mLatchForWaitingLoadingMainDictionaries.await(timeout, unit);
}
@UsedForTesting
public void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit)
throws InterruptedException {
waitForLoadingMainDictionaries(timeout, unit);
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final ExpandableBinaryDictionary dict : dictionaryGroup.mSubDictMap.values()) {
dict.waitAllTasksForTests();
}
}
}
public boolean isUserDictionaryEnabled() {
return mIsUserDictEnabled;
}
public void addWordToUserDictionary(final Context context, final String word) {
final Locale locale = getMostProbableLocale();
if (locale == null) {
return;
}
// TODO: add a toast telling what language this is being added to?
UserBinaryDictionary.addWordToUserDictionary(context, locale, word);
}
public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized,
@Nonnull final NgramContext ngramContext, final int timeStampInSeconds,
final boolean blockPotentiallyOffensive) {
final DictionaryGroup dictionaryGroup = getDictionaryGroupForMostProbableLanguage();
final String[] words = suggestion.split(Constants.WORD_SEPARATOR);
NgramContext ngramContextForCurrentWord = ngramContext;
for (int i = 0; i < words.length; i++) {
final String currentWord = words[i];
final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false;
addWordToUserHistory(dictionaryGroup, ngramContextForCurrentWord, currentWord,
wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive);
ngramContextForCurrentWord =
ngramContextForCurrentWord.getNextNgramContext(new WordInfo(currentWord));
}
}
private void addWordToUserHistory(final DictionaryGroup dictionaryGroup,
final NgramContext ngramContext, final String word, final boolean wasAutoCapitalized,
final int timeStampInSeconds, final boolean blockPotentiallyOffensive) {
final ExpandableBinaryDictionary userHistoryDictionary =
dictionaryGroup.getSubDict(Dictionary.TYPE_USER_HISTORY);
if (userHistoryDictionary == null
|| !isConfidentAboutCurrentLanguageBeing(userHistoryDictionary.mLocale)) {
return;
}
final int maxFreq = getFrequency(word);
if (maxFreq == 0 && blockPotentiallyOffensive) {
return;
}
final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale);
final String secondWord;
if (wasAutoCapitalized) {
if (isValidWord(word, false /* ignoreCase */)
&& !isValidWord(lowerCasedWord, false /* ignoreCase */)) {
// If the word was auto-capitalized and exists only as a capitalized word in the
// dictionary, then we must not downcase it before registering it. For example,
// the name of the contacts in start-of-sentence position would come here with the
// wasAutoCapitalized flag: if we downcase it, we'd register a lower-case version
// of that contact's name which would end up popping in suggestions.
secondWord = word;
} else {
// If however the word is not in the dictionary, or exists as a lower-case word
// only, then we consider that was a lower-case word that had been auto-capitalized.
secondWord = lowerCasedWord;
}
} else {
// HACK: We'd like to avoid adding the capitalized form of common words to the User
// History dictionary in order to avoid suggesting them until the dictionary
// consolidation is done.
// TODO: Remove this hack when ready.
final int lowerCaseFreqInMainDict = dictionaryGroup.hasDict(Dictionary.TYPE_MAIN,
null /* account */) ?
dictionaryGroup.getDict(Dictionary.TYPE_MAIN).getFrequency(lowerCasedWord) :
Dictionary.NOT_A_PROBABILITY;
if (maxFreq < lowerCaseFreqInMainDict
&& lowerCaseFreqInMainDict >= CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT) {
// Use lower cased word as the word can be a distracter of the popular word.
secondWord = lowerCasedWord;
} else {
secondWord = word;
}
}
// We demote unrecognized words (frequency < 0, below) by specifying them as "invalid".
// We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, ngramContext, secondWord,
isValid, timeStampInSeconds,
new DistracterFilterCheckingIsInDictionary(
mDistracterFilter, userHistoryDictionary));
}
private void removeWord(final String dictName, final String word) {
final ExpandableBinaryDictionary dictionary =
getDictionaryGroupForMostProbableLanguage().getSubDict(dictName);
if (dictionary != null) {
dictionary.removeUnigramEntryDynamically(word);
}
}
public void removeWordFromPersonalizedDicts(final String word) {
removeWord(Dictionary.TYPE_USER_HISTORY, word);
removeWord(Dictionary.TYPE_PERSONALIZATION, word);
removeWord(Dictionary.TYPE_CONTEXTUAL, word);
}
// TODO: Revise the way to fusion suggestion results.
public SuggestionResults getSuggestionResults(final WordComposer composer,
final NgramContext ngramContext, final long proximityInfoHandle,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId) {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
final SuggestionResults suggestionResults = new SuggestionResults(
SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext());
final float[] weightOfLangModelVsSpatialModel =
new float[] { Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL };
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
if (null == dictionary) continue;
final float weightForLocale = composer.isBatchMode()
? dictionaryGroup.mWeightForGesturingInLocale
: dictionaryGroup.mWeightForTypingInLocale;
final ArrayList<SuggestedWordInfo> dictionarySuggestions =
dictionary.getSuggestions(composer.getComposedDataSnapshot(), ngramContext,
proximityInfoHandle, settingsValuesForSuggestion, sessionId,
weightForLocale, weightOfLangModelVsSpatialModel);
if (null == dictionarySuggestions) continue;
suggestionResults.addAll(dictionarySuggestions);
if (null != suggestionResults.mRawSuggestions) {
suggestionResults.mRawSuggestions.addAll(dictionarySuggestions);
}
}
}
return suggestionResults;
}
public boolean isValidWord(final String word, final boolean ignoreCase) {
if (TextUtils.isEmpty(word)) {
return false;
}
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
if (dictionaryGroup.mLocale == null) {
continue;
}
final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale);
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
// Ideally the passed map would come out of a {@link java.util.concurrent.Future} and
// would be immutable once it's finished initializing, but concretely a null test is
// probably good enough for the time being.
if (null == dictionary) continue;
if (dictionary.isValidWord(word)
|| (ignoreCase && dictionary.isValidWord(lowerCasedWord))) {
return true;
}
}
}
return false;
}
private int getFrequencyInternal(final String word,
final boolean isGettingMaxFrequencyOfExactMatches) {
if (TextUtils.isEmpty(word)) {
return Dictionary.NOT_A_PROBABILITY;
}
int maxFreq = Dictionary.NOT_A_PROBABILITY;
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType);
if (dictionary == null) continue;
final int tempFreq;
if (isGettingMaxFrequencyOfExactMatches) {
tempFreq = dictionary.getMaxFrequencyOfExactMatches(word);
} else {
tempFreq = dictionary.getFrequency(word);
}
if (tempFreq >= maxFreq) {
maxFreq = tempFreq;
}
}
}
return maxFreq;
}
public int getFrequency(final String word) {
return getFrequencyInternal(word, false /* isGettingMaxFrequencyOfExactMatches */);
}
public int getMaxFrequencyOfExactMatches(final String word) {
return getFrequencyInternal(word, true /* isGettingMaxFrequencyOfExactMatches */);
}
private void clearSubDictionary(final String dictName) {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictName);
if (dictionary != null) {
dictionary.clear();
}
}
}
public void clearUserHistoryDictionary() {
clearSubDictionary(Dictionary.TYPE_USER_HISTORY);
}
// This method gets called only when the IME receives a notification to remove the
// personalization dictionary.
public void clearPersonalizationDictionary() {
clearSubDictionary(Dictionary.TYPE_PERSONALIZATION);
mPersonalizationHelper.clearDictionariesToUpdate();
}
public void clearContextualDictionary() {
clearSubDictionary(Dictionary.TYPE_CONTEXTUAL);
}
public void addEntriesToPersonalizationDictionary(
final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations,
final UpdateEntriesForInputEventsCallback callback) {
mPersonalizationHelper.updateEntriesOfPersonalizationDictionaries(
getMostProbableLocale(), personalizationDataChunk, spacingAndPunctuations,
callback);
}
@UsedForTesting
public void addPhraseToContextualDictionary(final String[] phrase, final int probability,
final int bigramProbabilityForWords, final int bigramProbabilityForPhrases) {
// TODO: we're inserting the phrase into the dictionary for the active language. Rethink
// this a bit from a theoretical point of view.
final ExpandableBinaryDictionary contextualDict =
getDictionaryGroupForMostProbableLanguage().getSubDict(Dictionary.TYPE_CONTEXTUAL);
if (contextualDict == null) {
return;
}
NgramContext ngramContext = NgramContext.BEGINNING_OF_SENTENCE;
for (int i = 0; i < phrase.length; i++) {
final String[] subPhrase = Arrays.copyOfRange(phrase, i /* start */, phrase.length);
final String subPhraseStr = TextUtils.join(Constants.WORD_SEPARATOR, subPhrase);
contextualDict.addUnigramEntryWithCheckingDistracter(
subPhraseStr, probability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY /* shortcutFreq */,
false /* isNotAWord */, false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
contextualDict.addNgramEntry(ngramContext, subPhraseStr,
bigramProbabilityForPhrases, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (i < phrase.length - 1) {
contextualDict.addUnigramEntryWithCheckingDistracter(
phrase[i], probability, null /* shortcutTarget */,
Dictionary.NOT_A_PROBABILITY /* shortcutFreq */,
false /* isNotAWord */, false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
contextualDict.addNgramEntry(ngramContext, phrase[i],
bigramProbabilityForWords, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
}
ngramContext =
ngramContext.getNextNgramContext(new NgramContext.WordInfo(phrase[i]));
}
}
public void dumpDictionaryForDebug(final String dictName) {
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
final ExpandableBinaryDictionary dictToDump = dictionaryGroup.getSubDict(dictName);
if (dictToDump == null) {
Log.e(TAG, "Cannot dump " + dictName + ". "
+ "The dictionary is not being used for suggestion or cannot be dumped.");
return;
}
dictToDump.dumpAllWordsForDebug();
}
}
public ArrayList<Pair<String, DictionaryStats>> getStatsOfEnabledSubDicts() {
final ArrayList<Pair<String, DictionaryStats>> statsOfEnabledSubDicts = new ArrayList<>();
final DictionaryGroup[] dictionaryGroups = mDictionaryGroups;
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : SUB_DICT_TYPES) {
final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictType);
if (dictionary == null) continue;
statsOfEnabledSubDicts.add(new Pair<>(dictType, dictionary.getDictionaryStats()));
}
}
return statsOfEnabledSubDicts;
}
}

2
java/src/com/android/inputmethod/latin/DictionaryFacilitatorLruCache.java
View File

@ -136,7 +136,7 @@ public class DictionaryFacilitatorLruCache {
if (dictionaryFacilitator != null) { if (dictionaryFacilitator != null) {
return dictionaryFacilitator; return dictionaryFacilitator;
} }
dictionaryFacilitator = new DictionaryFacilitator(); dictionaryFacilitator = DictionaryFacilitatorProvider.newDictionaryFacilitator();
resetDictionariesForLocaleLocked(dictionaryFacilitator, locale); resetDictionariesForLocaleLocked(dictionaryFacilitator, locale);
waitForLoadingMainDictionary(dictionaryFacilitator); waitForLoadingMainDictionary(dictionaryFacilitator);
mLruCache.put(locale, dictionaryFacilitator); mLruCache.put(locale, dictionaryFacilitator);

2
java/src/com/android/inputmethod/latin/LatinIME.java
View File

@ -140,7 +140,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final Settings mSettings; final Settings mSettings;
private final DictionaryFacilitator mDictionaryFacilitator = private final DictionaryFacilitator mDictionaryFacilitator =
new DictionaryFacilitator(this /* context */); DictionaryFacilitatorProvider.newDictionaryFacilitator(this /* context */);
// TODO: Move from LatinIME. // TODO: Move from LatinIME.
private final PersonalizationDictionaryUpdater mPersonalizationDictionaryUpdater = private final PersonalizationDictionaryUpdater mPersonalizationDictionaryUpdater =
new PersonalizationDictionaryUpdater(this /* context */, mDictionaryFacilitator); new PersonalizationDictionaryUpdater(this /* context */, mDictionaryFacilitator);

4
java/src/com/android/inputmethod/latin/settings/DebugSettingsFragment.java
View File

@ -28,7 +28,7 @@ import android.preference.PreferenceGroup;
import android.preference.TwoStatePreference; import android.preference.TwoStatePreference;
import com.android.inputmethod.latin.DictionaryDumpBroadcastReceiver; import com.android.inputmethod.latin.DictionaryDumpBroadcastReceiver;
import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.DictionaryFacilitatorImpl;
import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.R;
import com.android.inputmethod.latin.debug.ExternalDictionaryGetterForDebug; import com.android.inputmethod.latin.debug.ExternalDictionaryGetterForDebug;
import com.android.inputmethod.latin.utils.ApplicationUtils; import com.android.inputmethod.latin.utils.ApplicationUtils;
@ -67,7 +67,7 @@ public final class DebugSettingsFragment extends SubScreenFragment
final PreferenceGroup dictDumpPreferenceGroup = final PreferenceGroup dictDumpPreferenceGroup =
(PreferenceGroup)findPreference(PREF_KEY_DUMP_DICTS); (PreferenceGroup)findPreference(PREF_KEY_DUMP_DICTS);
for (final String dictName : DictionaryFacilitator.DICT_TYPE_TO_CLASS.keySet()) { for (final String dictName : DictionaryFacilitatorImpl.DICT_TYPE_TO_CLASS.keySet()) {
final Preference pref = new DictDumpPreference(getActivity(), dictName); final Preference pref = new DictDumpPreference(getActivity(), dictName);
pref.setOnPreferenceClickListener(this); pref.setOnPreferenceClickListener(this);
dictDumpPreferenceGroup.addPreference(pref); dictDumpPreferenceGroup.addPreference(pref);

4
tests/src/com/android/inputmethod/latin/personalization/ContextualDictionaryTests.java
View File

@ -25,6 +25,7 @@ import java.util.Map;
import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.DictionaryFacilitatorProvider;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import android.test.AndroidTestCase; import android.test.AndroidTestCase;
@ -40,7 +41,8 @@ public class ContextualDictionaryTests extends AndroidTestCase {
private DictionaryFacilitator getDictionaryFacilitator() { private DictionaryFacilitator getDictionaryFacilitator() {
final ArrayList<String> dictTypes = new ArrayList<>(); final ArrayList<String> dictTypes = new ArrayList<>();
dictTypes.add(Dictionary.TYPE_CONTEXTUAL); dictTypes.add(Dictionary.TYPE_CONTEXTUAL);
final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(); final DictionaryFacilitator dictionaryFacilitator =
DictionaryFacilitatorProvider.newDictionaryFacilitator();
dictionaryFacilitator.resetDictionariesForTesting(getContext(), dictionaryFacilitator.resetDictionariesForTesting(getContext(),
new Locale[] { LOCALE_EN_US }, dictTypes, new HashMap<String, File>(), new Locale[] { LOCALE_EN_US }, dictTypes, new HashMap<String, File>(),
Collections.<String, Map<String, String>>emptyMap(), null /* account */); Collections.<String, Map<String, String>>emptyMap(), null /* account */);

4
tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java
View File

@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.DictionaryFacilitatorProvider;
import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.RichInputMethodManager; import com.android.inputmethod.latin.RichInputMethodManager;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.UpdateEntriesForInputEventsCallback; import com.android.inputmethod.latin.ExpandableBinaryDictionary.UpdateEntriesForInputEventsCallback;
@ -55,7 +56,8 @@ public class PersonalizationDictionaryTests extends AndroidTestCase {
final ArrayList<String> dictTypes = new ArrayList<>(); final ArrayList<String> dictTypes = new ArrayList<>();
dictTypes.add(Dictionary.TYPE_MAIN); dictTypes.add(Dictionary.TYPE_MAIN);
dictTypes.add(Dictionary.TYPE_PERSONALIZATION); dictTypes.add(Dictionary.TYPE_PERSONALIZATION);
final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(getContext()); final DictionaryFacilitator dictionaryFacilitator =
DictionaryFacilitatorProvider.newDictionaryFacilitator(getContext());
dictionaryFacilitator.resetDictionariesForTesting(getContext(), dictionaryFacilitator.resetDictionariesForTesting(getContext(),
new Locale[] { LOCALE_EN_US }, dictTypes, new HashMap<String, File>(), new Locale[] { LOCALE_EN_US }, dictTypes, new HashMap<String, File>(),
Collections.<String, Map<String, String>>emptyMap(), null /* account */); Collections.<String, Map<String, String>>emptyMap(), null /* account */);