nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 71c795d0: Merge "Apply distracter filter for UserHistoryDictionary." 

* commit '71c795d00b7bc22a000b1cca50c53a365614a8eb':
  Apply distracter filter for UserHistoryDictionary.
main
Keisuke Kuroyanagi 2014-05-26 16:29:34 +00:00 committed by Android Git Automerger
parent f2da26d11d 71c795d00b
commit 0e45f98257
6 changed files with 45 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/src/com/android/inputmethod/latin/DictionaryFacilitator.java
View File

@ -168,7 +168,7 @@ public class DictionaryFacilitator {
}
public DictionaryFacilitator() {
mDistracterFilter = new DistracterFilter.EmptyDistracterFilter();
mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
}
public DictionaryFacilitator(final DistracterFilter distracterFilter) {
@ -448,7 +448,7 @@ public class DictionaryFacilitator {
// We don't add words with 0-frequency (assuming they would be profanity etc.).
final boolean isValid = maxFreq > 0;
UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord,
isValid, timeStampInSeconds);
isValid, timeStampInSeconds, mDistracterFilter);
}
public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo,

11
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.CombinedFormatUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.ExecutorUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
@ -271,9 +272,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Adds unigram information of a word to the dictionary. May overwrite an existing entry.
*/
public void addUnigramEntry(final String word, final int frequency,
public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency,
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
final boolean isBlacklisted, final int timestamp,
final DistracterFilter distracterFilter) {
reloadDictionaryIfRequired();
asyncExecuteTaskWithWriteLock(new Runnable() {
@Override
@ -281,6 +283,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (mBinaryDictionary == null) {
return;
}
if (distracterFilter.isDistracterToWordsInDictionaries(
PrevWordsInfo.EMPTY_PREV_WORDS_INFO, word, mLocale)) {
// The word is a distracter.
return;
}
runGCIfRequiredLocked(true /* mindsBlockByGC */);
addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq,
isNotAWord, isBlacklisted, timestamp);

9
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
View File

@ -23,6 +23,7 @@ import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.DistracterFilter;
import java.io.File;
import java.util.Locale;
@ -60,10 +61,11 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
* @param word the word the user inputted
* @param isValid whether the word is valid or not
* @param timestamp the timestamp when the word has been inputted
* @param distracterFilter the filter to check whether the word is a distracter
*/
public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary,
final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid,
final int timestamp) {
final int timestamp, final DistracterFilter distracterFilter) {
final String prevWord = prevWordsInfo.mPrevWord;
if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
@ -71,8 +73,9 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
}
final int frequency = isValid ?
FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
userHistoryDictionary.addUnigramEntry(word, frequency, null /* shortcutTarget */,
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp);
userHistoryDictionary.addUnigramEntryWithCheckingDistracter(word, frequency,
null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */,
false /* isBlacklisted */, timestamp, distracterFilter);
// Do not insert a word as a bigram of itself
if (word.equals(prevWord)) {
return;

4
java/src/com/android/inputmethod/latin/utils/DistracterFilter.java
View File

@ -40,7 +40,7 @@ public interface DistracterFilter {
public void close();
public static final class EmptyDistracterFilter implements DistracterFilter {
public static final DistracterFilter EMPTY_DISTRACTER_FILTER = new DistracterFilter() {
@Override
public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo,
String testedWord, Locale locale) {
@ -54,5 +54,5 @@ public interface DistracterFilter {
@Override
public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) {
}
}
};
}

26
java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
View File

@ -32,6 +32,7 @@ import android.view.inputmethod.InputMethodSubtype;
import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId;
import com.android.inputmethod.keyboard.KeyboardLayoutSet;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
@ -205,14 +206,25 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
final String consideredWord = trailingSingleQuotesCount > 0 ?
testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
testedWord;
final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults(
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, mKeyboard.getProximityInfo(),
true /* blockOffensiveWords */, null /* additionalFeaturesOptions */,
0 /* sessionId */, null /* rawSuggestions */);
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
@Override
public void run() {
final SuggestionResults suggestionResults =
mDictionaryFacilitator.getSuggestionResults(
composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
null /* additionalFeaturesOptions */, 0 /* sessionId */,
null /* rawSuggestions */);
if (suggestionResults.isEmpty()) {
return false;
holder.set(false);
return;
}
return isDistracter(suggestionResults, consideredWord);
holder.set(isDistracter(suggestionResults, consideredWord));
}
});
// It's OK to block the distracter filtering, but the dictionary lookup should be done
// sequentially using ExecutorUtils.
return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
}
}

7
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -23,6 +23,7 @@ import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import com.android.inputmethod.latin.utils.FileUtils;
import java.io.File;
@ -112,7 +113,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO;
for (String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true,
(int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()));
(int)TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
DistracterFilter.EMPTY_DISTRACTER_FILTER);
prevWordsInfo = new PrevWordsInfo(word);
}
}
@ -262,7 +264,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
dict.waitAllTasksForTests();
PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null);
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime);
UserHistoryDictionary.addToDictionary(dict, prevWordsInfo, word, true, mCurrentTime,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
prevWordsInfo = new PrevWordsInfo(word);
dict.waitAllTasksForTests();
assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word));