nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 89e34169: Get locale using detected language for personalization.

Browse source 
* commit '89e34169f825554fcf9325f855af302c5f63e785':
  Get locale using detected language for personalization.
This commit is contained in:
Keisuke Kuroyanagi 2014-09-10 09:36:09 +00:00 committed by Android Git Automerger
commit a8af53329e
3 changed files with 208 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
View file

@ -24,6 +24,7 @@ import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
import com.android.inputmethod.latin.PrevWordsInfo.WordInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.ContextualDictionary;
@ -36,7 +37,6 @@ import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.SuggestionResults;
import java.io.File;
@ -67,6 +67,7 @@ public class DictionaryFacilitator {
// To synchronize assigning mDictionaryGroup to ensure closing dictionaries.
private final Object mLock = new Object();
private final DistracterFilter mDistracterFilter;
private final PersonalizationDictionaryFacilitator mPersonalizationDictionaryFacilitator;
private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
new String[] {
@ -174,14 +175,18 @@ public class DictionaryFacilitator {
public DictionaryFacilitator() {
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
mPersonalizationDictionaryFacilitator = null;
}
public DictionaryFacilitator(final Context context) {
mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);
mPersonalizationDictionaryFacilitator =
new PersonalizationDictionaryFacilitator(context, mDistracterFilter);
}
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
mDistracterFilter.updateEnabledSubtypes(enabledSubtypes);
mPersonalizationDictionaryFacilitator.updateEnabledSubtypes(enabledSubtypes);
}
public Locale getLocale() {
@ -353,6 +358,9 @@ public class DictionaryFacilitator {
dictionaryGroup.closeDict(dictType);
}
mDistracterFilter.close();
if (mPersonalizationDictionaryFacilitator != null) {
mPersonalizationDictionaryFacilitator.close();
}
}
@UsedForTesting
@ -372,11 +380,11 @@ public class DictionaryFacilitator {
}
public void flushPersonalizationDictionary() {
final ExpandableBinaryDictionary personalizationDict =
final ExpandableBinaryDictionary personalizationDictUsedForSuggestion =
mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
if (personalizationDict != null) {
personalizationDict.asyncFlushBinaryDictionary();
}
mPersonalizationDictionaryFacilitator.flushPersonalizationDictionariesToUpdate(
personalizationDictUsedForSuggestion);
mDistracterFilter.close();
}
public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit)
@ -580,6 +588,7 @@ public class DictionaryFacilitator {
// personalization dictionary.
public void clearPersonalizationDictionary() {
clearSubDictionary(Dictionary.TYPE_PERSONALIZATION);
mPersonalizationDictionaryFacilitator.clearDictionariesToUpdate();
}
public void clearContextualDictionary() {
@ -589,30 +598,9 @@ public class DictionaryFacilitator {
public void addEntriesToPersonalizationDictionary(
final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations,
final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
final ExpandableBinaryDictionary personalizationDict =
mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
if (personalizationDict == null) {
if (callback != null) {
callback.onFinished();
}
return;
}
// TODO: Get locale from personalizationDataChunk.mDetectedLanguage.
final Locale dataChunkLocale = getLocale();
final ArrayList<LanguageModelParam> languageModelParams =
LanguageModelParam.createLanguageModelParamsFrom(
personalizationDataChunk.mTokens,
personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
dataChunkLocale, new DistracterFilterCheckingIsInDictionary(
mDistracterFilter, personalizationDict));
if (languageModelParams == null || languageModelParams.isEmpty()) {
if (callback != null) {
callback.onFinished();
}
return;
}
personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
final AddMultipleDictionaryEntriesCallback callback) {
mPersonalizationDictionaryFacilitator.addEntriesToPersonalizationDictionariesToUpdate(
personalizationDataChunk, spacingAndPunctuations, callback);
}
public void addPhraseToContextualDictionary(final String[] phrase, final int probability,

174
java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java Normal file
View file

@ -0,0 +1,174 @@
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicInteger;
import android.content.Context;
import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
import com.android.inputmethod.latin.personalization.PersonalizationDataChunk;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.SubtypeLocaleUtils;
/**
* Class for managing and updating personalization dictionaries.
*/
public class PersonalizationDictionaryFacilitator {
private final Context mContext;
private final DistracterFilter mDistracterFilter;
private final HashMap<String, HashSet<Locale>> mLangToLocalesMap = new HashMap<>();
private final HashMap<Locale, ExpandableBinaryDictionary> mPersonalizationDictsToUpdate =
new HashMap<>();
PersonalizationDictionaryFacilitator(final Context context,
final DistracterFilter distracterFilter) {
mContext = context;
mDistracterFilter = distracterFilter;
}
public void close() {
mLangToLocalesMap.clear();
for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
dict.close();
}
mPersonalizationDictsToUpdate.clear();
}
public void clearDictionariesToUpdate() {
for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
dict.clear();
}
mPersonalizationDictsToUpdate.clear();
}
public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
for (final InputMethodSubtype subtype : enabledSubtypes) {
final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
final String language = locale.getLanguage();
final HashSet<Locale> locales = mLangToLocalesMap.get(language);
if (locales != null) {
locales.add(locale);
} else {
final HashSet<Locale> localeSet = new HashSet<>();
localeSet.add(locale);
mLangToLocalesMap.put(language, localeSet);
}
}
}
/**
* Flush personalization dictionaries to dictionary files. Close dictionaries after writing
* files except the dictionary that is used for generating suggestions.
*
* @param personalizationDictUsedForSuggestion the personalization dictionary used for
* generating suggestions that won't be closed.
*/
public void flushPersonalizationDictionariesToUpdate(
final ExpandableBinaryDictionary personalizationDictUsedForSuggestion) {
for (final ExpandableBinaryDictionary personalizationDict :
mPersonalizationDictsToUpdate.values()) {
personalizationDict.asyncFlushBinaryDictionary();
if (personalizationDict != personalizationDictUsedForSuggestion) {
// Close if the dictionary is not being used for suggestion.
personalizationDict.close();
}
}
mDistracterFilter.close();
mPersonalizationDictsToUpdate.clear();
}
private ExpandableBinaryDictionary getPersonalizationDictToUpdate(final Context context,
final Locale locale) {
ExpandableBinaryDictionary personalizationDict = mPersonalizationDictsToUpdate.get(locale);
if (personalizationDict != null) {
return personalizationDict;
}
personalizationDict = PersonalizationDictionary.getDictionary(context, locale,
null /* dictFile */, "" /* dictNamePrefix */);
mPersonalizationDictsToUpdate.put(locale, personalizationDict);
return personalizationDict;
}
private void addEntriesToPersonalizationDictionariesForLocale(final Locale locale,
final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations,
final AddMultipleDictionaryEntriesCallback callback) {
final ExpandableBinaryDictionary personalizationDict =
getPersonalizationDictToUpdate(mContext, locale);
if (personalizationDict == null) {
if (callback != null) {
callback.onFinished();
}
return;
}
final ArrayList<LanguageModelParam> languageModelParams =
LanguageModelParam.createLanguageModelParamsFrom(
personalizationDataChunk.mTokens,
personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
locale, new DistracterFilterCheckingIsInDictionary(
mDistracterFilter, personalizationDict));
if (languageModelParams == null || languageModelParams.isEmpty()) {
if (callback != null) {
callback.onFinished();
}
return;
}
personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
}
public void addEntriesToPersonalizationDictionariesToUpdate(
final PersonalizationDataChunk personalizationDataChunk,
final SpacingAndPunctuations spacingAndPunctuations,
final AddMultipleDictionaryEntriesCallback callback) {
final HashSet<Locale> locales =
mLangToLocalesMap.get(personalizationDataChunk.mDetectedLanguage);
if (locales == null || locales.isEmpty()) {
if (callback != null) {
callback.onFinished();
}
return;
}
final AtomicInteger remainingTaskCount = new AtomicInteger(locales.size());
final AddMultipleDictionaryEntriesCallback callbackForLocales =
new AddMultipleDictionaryEntriesCallback() {
@Override
public void onFinished() {
if (remainingTaskCount.decrementAndGet() == 0) {
// Update tasks for all locales have been finished.
if (callback != null) {
callback.onFinished();
}
}
}
};
for (final Locale locale : locales) {
addEntriesToPersonalizationDictionariesForLocale(locale, personalizationDataChunk,
spacingAndPunctuations, callbackForLocales);
}
}
}

20
tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java
View file

@ -29,6 +29,7 @@ import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.RichInputMethodManager;
import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
@ -36,6 +37,7 @@ import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import android.view.inputmethod.InputMethodSubtype;
/**
* Unit tests for personalization dictionary
@ -55,16 +57,28 @@ public class PersonalizationDictionaryTests extends AndroidTestCase {
final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(getContext());
dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes,
new HashMap<String, File>(), new HashMap<String, Map<String, String>>());
// Set subtypes.
RichInputMethodManager.init(getContext());
final RichInputMethodManager richImm = RichInputMethodManager.getInstance();
final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>();
subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet(
LOCALE_EN_US.toString(), "qwerty"));
dictionaryFacilitator.updateEnabledSubtypes(subtypes);
return dictionaryFacilitator;
}
public void testAddManyTokens() {
final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator();
dictionaryFacilitator.clearPersonalizationDictionary();
final int dataChunkCount = 20;
final int wordCountInOneChunk = 2000;
final int dataChunkCount = 100;
final int wordCountInOneChunk = 200;
final int uniqueWordCount = 100;
final Random random = new Random(System.currentTimeMillis());
final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER;
final ArrayList<String> words = new ArrayList<>();
for (int i = 0; i < uniqueWordCount; i++) {
words.add(CodePointUtils.generateWord(random, codePointSet));
}
final SpacingAndPunctuations spacingAndPunctuations =
new SpacingAndPunctuations(getContext().getResources());
@ -75,7 +89,7 @@ public class PersonalizationDictionaryTests extends AndroidTestCase {
for (int i = 0; i < dataChunkCount; i++) {
final ArrayList<String> tokens = new ArrayList<>();
for (int j = 0; j < wordCountInOneChunk; j++) {
tokens.add(CodePointUtils.generateWord(random, codePointSet));
tokens.add(words.get(random.nextInt(words.size())));
}
final PersonalizationDataChunk personalizationDataChunk = new PersonalizationDataChunk(
true /* inputByUser */, tokens, timeStampInSeconds, DUMMY_PACKAGE_NAME,