Merge "Use DictionaryFacilitatorLruCache for personalization."

main
Keisuke Kuroyanagi 2014-08-29 10:10:39 +00:00 committed by Android (Google) Code Review
commit 9eec97d5b0
4 changed files with 57 additions and 39 deletions

View File

@ -33,6 +33,7 @@ import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.LanguageModelParam;
@ -59,6 +60,7 @@ public class DictionaryFacilitator {
// HACK: This threshold is being used when adding a capitalized entry in the User History // HACK: This threshold is being used when adding a capitalized entry in the User History
// dictionary. // dictionary.
private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140; private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140;
private static final int MAX_DICTIONARY_FACILITATOR_CACHE_SIZE = 3;
private Dictionaries mDictionaries = new Dictionaries(); private Dictionaries mDictionaries = new Dictionaries();
private boolean mIsUserDictEnabled = false; private boolean mIsUserDictEnabled = false;
@ -66,6 +68,7 @@ public class DictionaryFacilitator {
// To synchronize assigning mDictionaries to ensure closing dictionaries. // To synchronize assigning mDictionaries to ensure closing dictionaries.
private final Object mLock = new Object(); private final Object mLock = new Object();
private final DistracterFilter mDistracterFilter; private final DistracterFilter mDistracterFilter;
private final DictionaryFacilitatorLruCache mFacilitatorCacheForPersonalization;
private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS = private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
new String[] { new String[] {
@ -173,10 +176,14 @@ public class DictionaryFacilitator {
public DictionaryFacilitator() { public DictionaryFacilitator() {
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
mFacilitatorCacheForPersonalization = null;
} }
public DictionaryFacilitator(final DistracterFilter distracterFilter) { public DictionaryFacilitator(final Context context) {
mDistracterFilter = distracterFilter; mFacilitatorCacheForPersonalization = new DictionaryFacilitatorLruCache(context,
MAX_DICTIONARY_FACILITATOR_CACHE_SIZE, "" /* dictionaryNamePrefix */);
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context,
mFacilitatorCacheForPersonalization);
} }
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
@ -351,6 +358,9 @@ public class DictionaryFacilitator {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
dictionaries.closeDict(dictType); dictionaries.closeDict(dictType);
} }
if (mFacilitatorCacheForPersonalization != null) {
mFacilitatorCacheForPersonalization.evictAll();
}
mDistracterFilter.close(); mDistracterFilter.close();
} }
@ -597,11 +607,15 @@ public class DictionaryFacilitator {
} }
return; return;
} }
// TODO: Get locale from personalizationDataChunk.mDetectedLanguage.
final Locale dataChunkLocale = getLocale();
final DictionaryFacilitator dictionaryFacilitatorForLocale =
mFacilitatorCacheForPersonalization.get(dataChunkLocale);
final ArrayList<LanguageModelParam> languageModelParams = final ArrayList<LanguageModelParam> languageModelParams =
LanguageModelParam.createLanguageModelParamsFrom( LanguageModelParam.createLanguageModelParamsFrom(
personalizationDataChunk.mTokens, personalizationDataChunk.mTokens,
personalizationDataChunk.mTimestampInSeconds, personalizationDataChunk.mTimestampInSeconds,
this /* dictionaryFacilitator */, spacingAndPunctuations, dictionaryFacilitatorForLocale, spacingAndPunctuations,
new DistracterFilterCheckingIsInDictionary( new DistracterFilterCheckingIsInDictionary(
mDistracterFilter, personalizationDict)); mDistracterFilter, personalizationDict));
if (languageModelParams == null || languageModelParams.isEmpty()) { if (languageModelParams == null || languageModelParams.isEmpty()) {

View File

@ -131,8 +131,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
private final Settings mSettings; private final Settings mSettings;
private final DictionaryFacilitator mDictionaryFacilitator = private final DictionaryFacilitator mDictionaryFacilitator =
new DictionaryFacilitator( new DictionaryFacilitator(this /* context */);
new DistracterFilterCheckingExactMatchesAndSuggestions(this /* context */));
// TODO: Move from LatinIME. // TODO: Move from LatinIME.
private final PersonalizationDictionaryUpdater mPersonalizationDictionaryUpdater = private final PersonalizationDictionaryUpdater mPersonalizationDictionaryUpdater =
new PersonalizationDictionaryUpdater(this /* context */, mDictionaryFacilitator); new PersonalizationDictionaryUpdater(this /* context */, mDictionaryFacilitator);

View File

@ -20,7 +20,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit;
import android.content.Context; import android.content.Context;
import android.content.res.Resources; import android.content.res.Resources;
@ -34,6 +33,7 @@ import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId; import com.android.inputmethod.keyboard.KeyboardId;
import com.android.inputmethod.keyboard.KeyboardLayoutSet; import com.android.inputmethod.keyboard.KeyboardLayoutSet;
import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.DictionaryFacilitatorLruCache;
import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.RichInputMethodSubtype; import com.android.inputmethod.latin.RichInputMethodSubtype;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
@ -49,14 +49,15 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName(); DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName();
private static final boolean DEBUG = false; private static final boolean DEBUG = false;
private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;
private static final int MAX_DISTRACTERS_CACHE_SIZE = 512; private static final int MAX_DISTRACTERS_CACHE_SIZE = 512;
private final Context mContext; private final Context mContext;
private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap; private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
private final Map<Locale, Keyboard> mLocaleToKeyboardMap; private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
private final DictionaryFacilitator mDictionaryFacilitator; private final DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache;
private final LruCache<String, Boolean> mDistractersCache; private final LruCache<String, Boolean> mDistractersCache;
// TODO: Remove and support multiple locales at the same time.
private Locale mCurrentLocale;
private Keyboard mKeyboard; private Keyboard mKeyboard;
private final Object mLock = new Object(); private final Object mLock = new Object();
@ -71,19 +72,26 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
* Create a DistracterFilter instance. * Create a DistracterFilter instance.
* *
* @param context the context. * @param context the context.
* @param dictionaryFacilitatorLruCache the cache of dictionaryFacilitators that are used for
* checking distracters.
*/ */
public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context) { public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context,
final DictionaryFacilitatorLruCache dictionaryFacilitatorLruCache) {
mContext = context; mContext = context;
mLocaleToSubtypeMap = new HashMap<>(); mLocaleToSubtypeMap = new HashMap<>();
mLocaleToKeyboardMap = new HashMap<>(); mLocaleToKeyboardMap = new HashMap<>();
mDictionaryFacilitator = new DictionaryFacilitator(); mDictionaryFacilitatorLruCache = dictionaryFacilitatorLruCache;
mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE); mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE);
mCurrentLocale = null;
mKeyboard = null; mKeyboard = null;
} }
@Override @Override
public void close() { public void close() {
mDictionaryFacilitator.closeDictionaries(); mLocaleToKeyboardMap.clear();
mDistractersCache.evictAll();
mCurrentLocale = null;
mKeyboard = null;
} }
@Override @Override
@ -138,14 +146,6 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
} }
private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
mDictionaryFacilitator.resetDictionaries(mContext, newlocale,
false /* useContactsDict */, false /* usePersonalizedDicts */,
false /* forceReloadMainDictionary */, null /* listener */);
mDictionaryFacilitator.waitForLoadingMainDictionary(
TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS);
}
/** /**
* Determine whether a word is a distracter to words in dictionaries. * Determine whether a word is a distracter to words in dictionaries.
* *
@ -161,26 +161,20 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
if (locale == null) { if (locale == null) {
return false; return false;
} }
if (!locale.equals(mDictionaryFacilitator.getLocale())) { if (!locale.equals(mCurrentLocale)) {
synchronized (mLock) { synchronized (mLock) {
if (!mLocaleToSubtypeMap.containsKey(locale)) { if (!mLocaleToSubtypeMap.containsKey(locale)) {
Log.e(TAG, "Locale " + locale + " is not enabled."); Log.e(TAG, "Locale " + locale + " is not enabled.");
// TODO: Investigate what we should do for disabled locales. // TODO: Investigate what we should do for disabled locales.
return false; return false;
} }
mCurrentLocale = locale;
loadKeyboardForLocale(locale); loadKeyboardForLocale(locale);
// Reset dictionaries for the locale.
try {
mDistractersCache.evictAll(); mDistractersCache.evictAll();
loadDictionariesForLocale(locale);
} catch (final InterruptedException e) {
Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter",
e);
return false;
} }
} }
} final DictionaryFacilitator dictionaryFacilitator =
mDictionaryFacilitatorLruCache.get(locale);
if (DEBUG) { if (DEBUG) {
Log.d(TAG, "testedWord: " + testedWord); Log.d(TAG, "testedWord: " + testedWord);
} }
@ -193,13 +187,13 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
} }
final boolean isDistracterCheckedByGetMaxFreqencyOfExactMatches = final boolean isDistracterCheckedByGetMaxFreqencyOfExactMatches =
checkDistracterUsingMaxFreqencyOfExactMatches(testedWord); checkDistracterUsingMaxFreqencyOfExactMatches(dictionaryFacilitator, testedWord);
if (isDistracterCheckedByGetMaxFreqencyOfExactMatches) { if (isDistracterCheckedByGetMaxFreqencyOfExactMatches) {
// Add the word to the cache. // Add the word to the cache.
mDistractersCache.put(testedWord, Boolean.TRUE); mDistractersCache.put(testedWord, Boolean.TRUE);
return true; return true;
} }
final boolean isValidWord = mDictionaryFacilitator.isValidWord(testedWord, final boolean isValidWord = dictionaryFacilitator.isValidWord(testedWord,
false /* ignoreCase */); false /* ignoreCase */);
if (isValidWord) { if (isValidWord) {
// Valid word is not a distractor. // Valid word is not a distractor.
@ -210,7 +204,7 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
} }
final boolean isDistracterCheckedByGetSuggestion = final boolean isDistracterCheckedByGetSuggestion =
checkDistracterUsingGetSuggestions(testedWord); checkDistracterUsingGetSuggestions(dictionaryFacilitator, testedWord);
if (isDistracterCheckedByGetSuggestion) { if (isDistracterCheckedByGetSuggestion) {
// Add the word to the cache. // Add the word to the cache.
mDistractersCache.put(testedWord, Boolean.TRUE); mDistractersCache.put(testedWord, Boolean.TRUE);
@ -219,11 +213,12 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
return false; return false;
} }
private boolean checkDistracterUsingMaxFreqencyOfExactMatches(final String testedWord) { private static boolean checkDistracterUsingMaxFreqencyOfExactMatches(
final DictionaryFacilitator dictionaryFacilitator, final String testedWord) {
// The tested word is a distracter when there is a word that is exact matched to the tested // The tested word is a distracter when there is a word that is exact matched to the tested
// word and its probability is higher than the tested word's probability. // word and its probability is higher than the tested word's probability.
final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord); final int perfectMatchFreq = dictionaryFacilitator.getFrequency(testedWord);
final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord); final int exactMatchFreq = dictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
final boolean isDistracter = perfectMatchFreq < exactMatchFreq; final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
if (DEBUG) { if (DEBUG) {
Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq); Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
@ -233,7 +228,8 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
return isDistracter; return isDistracter;
} }
private boolean checkDistracterUsingGetSuggestions(final String testedWord) { private boolean checkDistracterUsingGetSuggestions(
final DictionaryFacilitator dictionaryFacilitator, final String testedWord) {
if (mKeyboard == null) { if (mKeyboard == null) {
return false; return false;
} }
@ -251,7 +247,7 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr
synchronized (mLock) { synchronized (mLock) {
final int[] coordinates = mKeyboard.getCoordinates(codePoints); final int[] coordinates = mKeyboard.getCoordinates(codePoints);
composer.setComposingWord(codePoints, coordinates); composer.setComposingWord(codePoints, coordinates);
final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( final SuggestionResults suggestionResults = dictionaryFacilitator.getSuggestionResults(
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, mKeyboard.getProximityInfo(), composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, mKeyboard.getProximityInfo(),
settingsValuesForSuggestion, 0 /* sessionId */); settingsValuesForSuggestion, 0 /* sessionId */);
if (suggestionResults.isEmpty()) { if (suggestionResults.isEmpty()) {

View File

@ -31,13 +31,17 @@ import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesA
*/ */
@LargeTest @LargeTest
public class DistracterFilterTest extends AndroidTestCase { public class DistracterFilterTest extends AndroidTestCase {
private DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache;
private DistracterFilterCheckingExactMatchesAndSuggestions mDistracterFilter; private DistracterFilterCheckingExactMatchesAndSuggestions mDistracterFilter;
@Override @Override
protected void setUp() throws Exception { protected void setUp() throws Exception {
super.setUp(); super.setUp();
final Context context = getContext(); final Context context = getContext();
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); mDictionaryFacilitatorLruCache = new DictionaryFacilitatorLruCache(context,
2 /* maxSize */, "" /* dictionaryNamePrefix */);
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context,
mDictionaryFacilitatorLruCache);
RichInputMethodManager.init(context); RichInputMethodManager.init(context);
final RichInputMethodManager richImm = RichInputMethodManager.getInstance(); final RichInputMethodManager richImm = RichInputMethodManager.getInstance();
final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>(); final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>();
@ -50,6 +54,11 @@ public class DistracterFilterTest extends AndroidTestCase {
mDistracterFilter.updateEnabledSubtypes(subtypes); mDistracterFilter.updateEnabledSubtypes(subtypes);
} }
@Override
protected void tearDown() {
mDictionaryFacilitatorLruCache.evictAll();
}
public void testIsDistractorToWordsInDictionaries() { public void testIsDistractorToWordsInDictionaries() {
final PrevWordsInfo EMPTY_PREV_WORDS_INFO = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; final PrevWordsInfo EMPTY_PREV_WORDS_INFO = PrevWordsInfo.EMPTY_PREV_WORDS_INFO;