From 83c40a2301a0b5a42a75eecada48e7887a7c940e Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Mon, 19 May 2014 13:55:40 +0900 Subject: [PATCH] Use PrevWordsInfo instead of String in Java side. Bug: 14119293 Bug: 14425059 Change-Id: I3d5da84881a49a04550180dd9aac2c37da2ed762 --- .../inputmethod/latin/BinaryDictionary.java | 6 ++-- .../android/inputmethod/latin/Dictionary.java | 8 ++--- .../latin/DictionaryCollection.java | 6 ++-- .../DictionaryFacilitatorForSuggest.java | 4 +-- .../latin/ExpandableBinaryDictionary.java | 4 +-- .../android/inputmethod/latin/LatinIME.java | 10 +++---- .../inputmethod/latin/PrevWordsInfo.java | 29 +++++++++++++++++++ .../latin/ReadOnlyBinaryDictionary.java | 4 +-- .../android/inputmethod/latin/Suggest.java | 16 +++++----- .../inputmethod/latin/WordComposer.java | 28 +++++++++--------- .../AndroidSpellCheckerSession.java | 8 +++-- .../AndroidWordLevelSpellCheckerSession.java | 29 ++++++++++--------- .../latin/spellcheck/DictionaryPool.java | 3 +- ...onouslyLoadedContactsBinaryDictionary.java | 5 ++-- ...nchronouslyLoadedUserBinaryDictionary.java | 5 ++-- .../latin/utils/DistracterFilter.java | 9 +++--- .../latin/utils/LanguageModelParam.java | 29 ++++++++++--------- .../latin/DistracterFilterTest.java | 18 ++++++------ .../inputmethod/latin/WordComposerTests.java | 16 +++++----- tools/dicttool/Android.mk | 1 + 20 files changed, 137 insertions(+), 101 deletions(-) create mode 100644 java/src/com/android/inputmethod/latin/PrevWordsInfo.java diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 8a6404b1b..b8cf3f89c 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -265,7 +265,7 @@ public final class BinaryDictionary extends Dictionary { @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { if (!isValidDictionary()) { @@ -274,8 +274,8 @@ public final class BinaryDictionary extends Dictionary { Arrays.fill(mInputCodePoints, Constants.NOT_A_CODE); // TODO: toLowerCase in the native code - final int[] prevWordCodePointArray = (null == prevWord) - ? null : StringUtils.toCodePointArray(prevWord); + final int[] prevWordCodePointArray = (null == prevWordsInfo.mPrevWord) + ? null : StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); final InputPointers inputPointers = composer.getInputPointers(); final boolean isGesture = composer.isBatchMode(); final int inputSize; diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 5253cc3dd..aab16653e 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -69,7 +69,7 @@ public abstract class Dictionary { * Searches for suggestions for a given context. For the moment the context is only the * previous word. * @param composer the key sequence to match with coordinate info, as a WordComposer - * @param prevWord the previous word, or null if none + * @param prevWordsInfo the information of previous words. * @param proximityInfo the object for key proximity. May be ignored by some implementations. * @param blockOffensiveWords whether to block potentially offensive words * @param additionalFeaturesOptions options about additional features used for the suggestion. @@ -79,10 +79,8 @@ public abstract class Dictionary { * different language weight is used. * @return the list of suggestions (possibly null if none) */ - // TODO: pass more context than just the previous word, to enable better suggestions (n-gram - // and more) abstract public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight); @@ -156,7 +154,7 @@ public abstract class Dictionary { @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { return null; diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java index 239fd067a..e6e4e0938 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java +++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java @@ -57,7 +57,7 @@ public final class DictionaryCollection extends Dictionary { @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { final CopyOnWriteArrayList dictionaries = mDictionaries; @@ -65,13 +65,13 @@ public final class DictionaryCollection extends Dictionary { // To avoid creating unnecessary objects, we get the list out of the first // dictionary and add the rest to it if not null, hence the get(0) ArrayList suggestions = dictionaries.get(0).getSuggestions(composer, - prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); if (null == suggestions) suggestions = CollectionUtils.newArrayList(); final int length = dictionaries.size(); for (int i = 1; i < length; ++ i) { final ArrayList sugg = dictionaries.get(i).getSuggestions(composer, - prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); if (null != sugg) suggestions.addAll(sugg); } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java index ddbb196d9..14c8bb6c3 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java @@ -444,7 +444,7 @@ public class DictionaryFacilitatorForSuggest { // TODO: Revise the way to fusion suggestion results. public SuggestionResults getSuggestionResults(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final ArrayList rawSuggestions) { final Dictionaries dictionaries = mDictionaries; @@ -455,7 +455,7 @@ public class DictionaryFacilitatorForSuggest { final Dictionary dictionary = dictionaries.getDict(dictType); if (null == dictionary) continue; final ArrayList dictionarySuggestions = - dictionary.getSuggestions(composer, prevWord, proximityInfo, + dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, languageWeight); if (null == dictionarySuggestions) continue; diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 4358f84c2..629f3fd18 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -367,7 +367,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { reloadDictionaryIfRequired(); @@ -380,7 +380,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return null; } final ArrayList suggestions = - mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, + mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); if (mBinaryDictionary.isCorrupted()) { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index d100d32bb..5e45275f8 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1428,8 +1428,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (DEBUG) { if (mInputLogic.mWordComposer.isComposingWord() || mInputLogic.mWordComposer.isBatchMode()) { - final String previousWord - = mInputLogic.mWordComposer.getPreviousWordForSuggestion(); + final PrevWordsInfo prevWordsInfo + = mInputLogic.mWordComposer.getPrevWordsInfoForSuggestion(); // TODO: this is for checking consistency with older versions. Remove this when // we are confident this is stable. // We're checking the previous word in the text field against the memorized previous @@ -1438,14 +1438,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final CharSequence rereadPrevWord = mInputLogic.getNthPreviousWordForSuggestion( currentSettings.mSpacingAndPunctuations, mInputLogic.mWordComposer.isComposingWord() ? 2 : 1); - if (!TextUtils.equals(previousWord, rereadPrevWord)) { + if (!TextUtils.equals(prevWordsInfo.mPrevWord, rereadPrevWord)) { throw new RuntimeException("Unexpected previous word: " - + previousWord + " <> " + rereadPrevWord); + + prevWordsInfo.mPrevWord + " <> " + rereadPrevWord); } } } mInputLogic.mSuggest.getSuggestedWords(mInputLogic.mWordComposer, - mInputLogic.mWordComposer.getPreviousWordForSuggestion(), + mInputLogic.mWordComposer.getPrevWordsInfoForSuggestion(), keyboard.getProximityInfo(), currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, additionalFeaturesOptions, sessionId, sequenceNumber, callback); diff --git a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java new file mode 100644 index 000000000..9d8543183 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +public class PrevWordsInfo { + // The previous word. May be null after resetting and before starting a new composing word, or + // when there is no context like at the start of text for example. It can also be set to null + // externally when the user enters a separator that does not let bigrams across, like a period + // or a comma. + public final String mPrevWord; + + public PrevWordsInfo(final String prevWord) { + mPrevWord = prevWord; + } +} diff --git a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java index 15b12381b..8f744bef8 100644 --- a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java @@ -50,12 +50,12 @@ public final class ReadOnlyBinaryDictionary extends Dictionary { @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { if (mLock.readLock().tryLock()) { try { - return mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, + return mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); } finally { diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 71355f418..e3759a586 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -71,17 +71,17 @@ public final class Suggest { } public void getSuggestedWords(final WordComposer wordComposer, - final String prevWordForBigram, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final boolean isCorrectionEnabled, final int[] additionalFeaturesOptions, final int sessionId, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { - LatinImeLogger.onStartSuggestion(prevWordForBigram); + LatinImeLogger.onStartSuggestion(prevWordsInfo.mPrevWord); if (wordComposer.isBatchMode()) { - getSuggestedWordsForBatchInput(wordComposer, prevWordForBigram, proximityInfo, + getSuggestedWordsForBatchInput(wordComposer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, sequenceNumber, callback); } else { - getSuggestedWordsForTypingInput(wordComposer, prevWordForBigram, proximityInfo, + getSuggestedWordsForTypingInput(wordComposer, prevWordsInfo, proximityInfo, blockOffensiveWords, isCorrectionEnabled, additionalFeaturesOptions, sequenceNumber, callback); } @@ -90,7 +90,7 @@ public final class Suggest { // Retrieves suggestions for the typing input // and calls the callback function with the suggestions. private void getSuggestedWordsForTypingInput(final WordComposer wordComposer, - final String prevWordForBigram, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final boolean isCorrectionEnabled, final int[] additionalFeaturesOptions, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { @@ -108,7 +108,7 @@ public final class Suggest { rawSuggestions = null; } final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( - wordComposer, prevWordForBigram, proximityInfo, blockOffensiveWords, + wordComposer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, SESSION_TYPING, rawSuggestions); final boolean isFirstCharCapitalized = wordComposer.isFirstCharCapitalized(); @@ -215,7 +215,7 @@ public final class Suggest { // Retrieves suggestions for the batch input // and calls the callback function with the suggestions. private void getSuggestedWordsForBatchInput(final WordComposer wordComposer, - final String prevWordForBigram, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { @@ -226,7 +226,7 @@ public final class Suggest { rawSuggestions = null; } final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( - wordComposer, prevWordForBigram, proximityInfo, blockOffensiveWords, + wordComposer, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, rawSuggestions); for (SuggestedWordInfo wordInfo : suggestionResults) { LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 9cf71c7f4..227b42bde 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -46,11 +46,9 @@ public final class WordComposer { // The list of events that served to compose this string. private final ArrayList mEvents; private final InputPointers mInputPointers = new InputPointers(MAX_WORD_LENGTH); - // The previous word (before the composing word). Used as context for suggestions. May be null - // after resetting and before starting a new composing word, or when there is no context like - // at the start of text for example. It can also be set to null externally when the user - // enters a separator that does not let bigrams across, like a period or a comma. - private String mPreviousWordForSuggestion; + // The information of previous words (before the composing word). Must not be null. Used as + // context for suggestions. + private PrevWordsInfo mPrevWordsInfo; private String mAutoCorrection; private boolean mIsResumed; private boolean mIsBatchMode; @@ -87,7 +85,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWordForSuggestion = null; + mPrevWordsInfo = new PrevWordsInfo(null); refreshTypedWordCache(); } @@ -119,7 +117,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWordForSuggestion = null; + mPrevWordsInfo = new PrevWordsInfo(null); refreshTypedWordCache(); } @@ -309,7 +307,7 @@ public final class WordComposer { CoordinateUtils.yFromArray(coordinates, i))); } mIsResumed = true; - mPreviousWordForSuggestion = null == previousWord ? null : previousWord.toString(); + mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString()); } /** @@ -320,8 +318,8 @@ public final class WordComposer { return mTypedWordCache.toString(); } - public String getPreviousWordForSuggestion() { - return mPreviousWordForSuggestion; + public PrevWordsInfo getPrevWordsInfoForSuggestion() { + return mPrevWordsInfo; } /** @@ -379,7 +377,7 @@ public final class WordComposer { public void setCapitalizedModeAndPreviousWordAtStartComposingTime(final int mode, final CharSequence previousWord) { mCapitalizedMode = mode; - mPreviousWordForSuggestion = null == previousWord ? null : previousWord.toString(); + mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString()); } /** @@ -430,7 +428,7 @@ public final class WordComposer { mCapsCount = 0; mDigitsCount = 0; mIsBatchMode = false; - mPreviousWordForSuggestion = committedWord.toString(); + mPrevWordsInfo = new PrevWordsInfo(committedWord.toString()); mCombinerChain.reset(); mEvents.clear(); mCodePointSize = 0; @@ -448,11 +446,11 @@ public final class WordComposer { // when the user inputs a separator that's not whitespace (including the case of the // double-space-to-period feature). public void discardPreviousWordForSuggestion() { - mPreviousWordForSuggestion = null; + mPrevWordsInfo = new PrevWordsInfo(null); } public void resumeSuggestionOnLastComposedWord(final LastComposedWord lastComposedWord, - final String previousWord) { + final PrevWordsInfo prevWordsInfo) { mEvents.clear(); Collections.copy(mEvents, lastComposedWord.mEvents); mInputPointers.set(lastComposedWord.mInputPointers); @@ -463,7 +461,7 @@ public final class WordComposer { mCursorPositionWithinWord = mCodePointSize; mRejectedBatchModeSuggestion = null; mIsResumed = true; - mPreviousWordForSuggestion = previousWord; + mPrevWordsInfo = prevWordsInfo; } public boolean isBatchMode() { diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java index ddda52d71..e951f5a89 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java @@ -23,6 +23,7 @@ import android.view.textservice.SentenceSuggestionsInfo; import android.view.textservice.SuggestionsInfo; import android.view.textservice.TextInfo; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.utils.CollectionUtils; import java.util.ArrayList; @@ -57,7 +58,7 @@ public final class AndroidSpellCheckerSession extends AndroidWordLevelSpellCheck final int offset = ssi.getOffsetAt(i); final int length = ssi.getLengthAt(i); final String subText = typedText.substring(offset, offset + length); - final String prevWord = currentWord; + final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(currentWord); currentWord = subText; if (!subText.contains(AndroidSpellCheckerService.SINGLE_QUOTE)) { continue; @@ -73,7 +74,7 @@ public final class AndroidSpellCheckerSession extends AndroidWordLevelSpellCheck if (TextUtils.isEmpty(splitText)) { continue; } - if (mSuggestionsCache.getSuggestionsFromCache(splitText, prevWord) == null) { + if (mSuggestionsCache.getSuggestionsFromCache(splitText, prevWordsInfo) == null) { continue; } final int newLength = splitText.length(); @@ -148,7 +149,8 @@ public final class AndroidSpellCheckerSession extends AndroidWordLevelSpellCheck } else { prevWord = null; } - retval[i] = onGetSuggestionsInternal(textInfos[i], prevWord, suggestionsLimit); + final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(prevWord); + retval[i] = onGetSuggestionsInternal(textInfos[i], prevWordsInfo, suggestionsLimit); retval[i].setCookieAndSequence(textInfos[i].getCookie(), textInfos[i].getSequence()); } diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java index 4c23d2210..cf26000d5 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java @@ -31,6 +31,7 @@ import com.android.inputmethod.compat.SuggestionsInfoCompatUtils; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; import com.android.inputmethod.latin.spellcheck.AndroidSpellCheckerService.SuggestionsGatherer; @@ -71,26 +72,26 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { new LruCache(MAX_CACHE_SIZE); // TODO: Support n-gram input - private static String generateKey(String query, String prevWord) { - if (TextUtils.isEmpty(query) || TextUtils.isEmpty(prevWord)) { + private static String generateKey(final String query, final PrevWordsInfo prevWordsInfo) { + if (TextUtils.isEmpty(query) || TextUtils.isEmpty(prevWordsInfo.mPrevWord)) { return query; } - return query + CHAR_DELIMITER + prevWord; + return query + CHAR_DELIMITER + prevWordsInfo.mPrevWord; } - // TODO: Support n-gram input - public SuggestionsParams getSuggestionsFromCache(String query, String prevWord) { - return mUnigramSuggestionsInfoCache.get(generateKey(query, prevWord)); + public SuggestionsParams getSuggestionsFromCache(String query, + final PrevWordsInfo prevWordsInfo) { + return mUnigramSuggestionsInfoCache.get(generateKey(query, prevWordsInfo)); } - // TODO: Support n-gram input public void putSuggestionsToCache( - String query, String prevWord, String[] suggestions, int flags) { + final String query, final PrevWordsInfo prevWordsInfo, + final String[] suggestions, final int flags) { if (suggestions == null || TextUtils.isEmpty(query)) { return; } mUnigramSuggestionsInfoCache.put( - generateKey(query, prevWord), new SuggestionsParams(suggestions, flags)); + generateKey(query, prevWordsInfo), new SuggestionsParams(suggestions, flags)); } public void clearCache() { @@ -259,11 +260,12 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { } protected SuggestionsInfo onGetSuggestionsInternal( - final TextInfo textInfo, final String prevWord, final int suggestionsLimit) { + final TextInfo textInfo, final PrevWordsInfo prevWordsInfo, + final int suggestionsLimit) { try { final String inText = textInfo.getText(); final SuggestionsParams cachedSuggestionsParams = - mSuggestionsCache.getSuggestionsFromCache(inText, prevWord); + mSuggestionsCache.getSuggestionsFromCache(inText, prevWordsInfo); if (cachedSuggestionsParams != null) { if (DBG) { Log.d(TAG, "Cache hit: " + inText + ", " + cachedSuggestionsParams.mFlags); @@ -325,7 +327,7 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { composer.setComposingWord(codePoints, coordinates, null /* previousWord */); // TODO: make a spell checker option to block offensive words or not final ArrayList suggestions = - dictInfo.mDictionary.getSuggestions(composer, prevWord, + dictInfo.mDictionary.getSuggestions(composer, prevWordsInfo, dictInfo.getProximityInfo(), true /* blockOffensiveWords */, null /* additionalFeaturesOptions */, 0 /* sessionId */, null /* inOutLanguageWeight */); @@ -369,7 +371,8 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { .getValueOf_RESULT_ATTR_HAS_RECOMMENDED_SUGGESTIONS() : 0); final SuggestionsInfo retval = new SuggestionsInfo(flags, result.mSuggestions); - mSuggestionsCache.putSuggestionsToCache(text, prevWord, result.mSuggestions, flags); + mSuggestionsCache.putSuggestionsToCache(text, prevWordsInfo, result.mSuggestions, + flags); return retval; } catch (RuntimeException e) { // Don't kill the keyboard if there is a bug in the spell checker diff --git a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java index 0be256805..ba2e0c309 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java @@ -20,6 +20,7 @@ import android.util.Log; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -52,7 +53,7 @@ public final class DictionaryPool extends LinkedBlockingQueue { // TODO: this dummy dictionary should be a singleton in the Dictionary class. @Override public ArrayList getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { return noSuggestions; diff --git a/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedContactsBinaryDictionary.java index 5f6e168b3..75075664f 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedContactsBinaryDictionary.java @@ -20,6 +20,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.ContactsBinaryDictionary; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; @@ -36,11 +37,11 @@ public final class SynchronouslyLoadedContactsBinaryDictionary extends ContactsB @Override public ArrayList getSuggestions(final WordComposer codes, - final String prevWordForBigrams, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { synchronized (mLock) { - return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, + return super.getSuggestions(codes, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); } } diff --git a/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedUserBinaryDictionary.java index 0499ad234..f2d981a9d 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedUserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/SynchronouslyLoadedUserBinaryDictionary.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.spellcheck; import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.UserBinaryDictionary; import com.android.inputmethod.latin.WordComposer; @@ -41,11 +42,11 @@ public final class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDic @Override public ArrayList getSuggestions(final WordComposer codes, - final String prevWordForBigrams, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { synchronized (mLock) { - return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, + return super.getSuggestions(codes, prevWordsInfo, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId, inOutLanguageWeight); } } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java index 05387d55b..a21953259 100644 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java @@ -24,6 +24,7 @@ import android.util.Log; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.Suggest; import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; import com.android.inputmethod.latin.SuggestedWords; @@ -88,13 +89,13 @@ public class DistracterFilter { /** * Determine whether a word is a distracter to words in dictionaries. * - * @param prevWord the previous word, or null if none. + * @param prevWordsInfo the information of previous words. * @param testedWord the word that will be tested to see whether it is a distracter to words * in dictionaries. * @param locale the locale of words. * @return true if testedWord is a distracter, otherwise false. */ - public boolean isDistracterToWordsInDictionaries(final String prevWord, + public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, final String testedWord, final Locale locale) { if (mKeyboard == null || locale == null) { return false; @@ -113,7 +114,7 @@ public class DistracterFilter { final int[] codePoints = StringUtils.toCodePointArray(testedWord); final int[] coordinates; coordinates = mKeyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates, prevWord); + composer.setComposingWord(codePoints, coordinates, prevWordsInfo.mPrevWord); final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); final String consideredWord = trailingSingleQuotesCount > 0 ? @@ -133,7 +134,7 @@ public class DistracterFilter { } } }; - mSuggest.getSuggestedWords(composer, prevWord, mKeyboard.getProximityInfo(), + mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, null /* additionalFeaturesOptions */, 0 /* sessionId */, SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java index 2d6796e3e..aaf4a4064 100644 --- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java +++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java @@ -20,6 +20,7 @@ import android.util.Log; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import java.util.ArrayList; @@ -85,7 +86,7 @@ public final class LanguageModelParam { final ArrayList languageModelParams = CollectionUtils.newArrayList(); final int N = tokens.size(); - String prevWord = null; + PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null); for (int i = 0; i < N; ++i) { final String tempWord = tokens.get(i); if (StringUtils.isEmptyStringOrWhiteSpaces(tempWord)) { @@ -102,7 +103,7 @@ public final class LanguageModelParam { + tempWord + "\""); } // Sentence terminator found. Split. - prevWord = null; + prevWordsInfo = new PrevWordsInfo(null); continue; } if (DEBUG_TOKEN) { @@ -110,19 +111,19 @@ public final class LanguageModelParam { } final LanguageModelParam languageModelParam = detectWhetherVaildWordOrNotAndGetLanguageModelParam( - prevWord, tempWord, timestamp, dictionaryFacilitator, + prevWordsInfo, tempWord, timestamp, dictionaryFacilitator, distracterFilter); if (languageModelParam == null) { continue; } languageModelParams.add(languageModelParam); - prevWord = languageModelParam.mTargetWord; + prevWordsInfo = new PrevWordsInfo(languageModelParam.mTargetWord); } return languageModelParams; } private static LanguageModelParam detectWhetherVaildWordOrNotAndGetLanguageModelParam( - final String prevWord, final String targetWord, final int timestamp, + final PrevWordsInfo prevWordsInfo, final String targetWord, final int timestamp, final DictionaryFacilitatorForSuggest dictionaryFacilitator, final DistracterFilter distracterFilter) { final Locale locale = dictionaryFacilitator.getLocale(); @@ -133,14 +134,14 @@ public final class LanguageModelParam { // distracterFilter in the following code. If targetWord is a distracter, // it should be filtered out. if (dictionaryFacilitator.isValidWord(targetWord, false /* ignoreCase */)) { - return createAndGetLanguageModelParamOfWord(prevWord, targetWord, timestamp, + return createAndGetLanguageModelParamOfWord(prevWordsInfo, targetWord, timestamp, true /* isValidWord */, locale); } final String lowerCaseTargetWord = targetWord.toLowerCase(locale); if (dictionaryFacilitator.isValidWord(lowerCaseTargetWord, false /* ignoreCase */)) { // Add the lower-cased word. - return createAndGetLanguageModelParamOfWord(prevWord, lowerCaseTargetWord, + return createAndGetLanguageModelParamOfWord(prevWordsInfo, lowerCaseTargetWord, timestamp, true /* isValidWord */, locale); } @@ -150,26 +151,26 @@ public final class LanguageModelParam { // Adding such a word to dictonaries would interfere with entering in-dictionary words. For // example, adding "mot" to dictionaries might interfere with entering "not". // This kind of OOV should be filtered out. - if (distracterFilter.isDistracterToWordsInDictionaries(prevWord, targetWord, locale)) { + if (distracterFilter.isDistracterToWordsInDictionaries(prevWordsInfo, targetWord, locale)) { return null; } - return createAndGetLanguageModelParamOfWord(prevWord, targetWord, timestamp, + return createAndGetLanguageModelParamOfWord(prevWordsInfo, targetWord, timestamp, false /* isValidWord */, locale); } private static LanguageModelParam createAndGetLanguageModelParamOfWord( - final String prevWord, final String targetWord, final int timestamp, + final PrevWordsInfo prevWordsInfo, final String targetWord, final int timestamp, final boolean isValidWord, final Locale locale) { final String word; if (StringUtils.getCapitalizationType(targetWord) == StringUtils.CAPITALIZE_FIRST - && prevWord == null && !isValidWord) { + && prevWordsInfo.mPrevWord == null && !isValidWord) { word = targetWord.toLowerCase(locale); } else { word = targetWord; } final int unigramProbability = isValidWord ? UNIGRAM_PROBABILITY_FOR_VALID_WORD : UNIGRAM_PROBABILITY_FOR_OOV_WORD; - if (prevWord == null) { + if (prevWordsInfo.mPrevWord == null) { if (DEBUG) { Log.d(TAG, "--- add unigram: current(" + (isValidWord ? "Valid" : "OOV") + ") = " + word); @@ -177,12 +178,12 @@ public final class LanguageModelParam { return new LanguageModelParam(word, unigramProbability, timestamp); } if (DEBUG) { - Log.d(TAG, "--- add bigram: prev = " + prevWord + ", current(" + Log.d(TAG, "--- add bigram: prev = " + prevWordsInfo.mPrevWord + ", current(" + (isValidWord ? "Valid" : "OOV") + ") = " + word); } final int bigramProbability = isValidWord ? BIGRAM_PROBABILITY_FOR_VALID_WORD : BIGRAM_PROBABILITY_FOR_OOV_WORD; - return new LanguageModelParam(prevWord, word, unigramProbability, + return new LanguageModelParam(prevWordsInfo.mPrevWord, word, unigramProbability, bigramProbability, timestamp); } } diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java index d7b57aeea..e98f9eacc 100644 --- a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java +++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java @@ -36,50 +36,50 @@ public class DistracterFilterTest extends InputTestsBase { } public void testIsDistractorToWordsInDictionaries() { - final String EMPTY_PREV_WORD = null; + final PrevWordsInfo EMPTY_PREV_WORDS_INFO = new PrevWordsInfo(null); final Locale localeEnUs = new Locale("en", "US"); String typedWord = "alot"; // For this test case, we consider "alot" is a distracter to "a lot". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "mot"; // For this test case, we consider "mot" is a distracter to "not". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "wierd"; // For this test case, we consider "wierd" is a distracter to "weird". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "hoe"; // For this test case, we consider "hoe" is a distracter to "how". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "nit"; // For this test case, we consider "nit" is a distracter to "not". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "ill"; // For this test case, we consider "ill" is a distracter to "I'll". assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "asdfd"; // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries. assertFalse( mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); typedWord = "thank"; // For this test case, we consider "thank" is not a distracter to any other word // in dictionaries. assertFalse( mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORD, typedWord, localeEnUs)); + EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); } } diff --git a/tests/src/com/android/inputmethod/latin/WordComposerTests.java b/tests/src/com/android/inputmethod/latin/WordComposerTests.java index d68bb5c54..16e8b36b0 100644 --- a/tests/src/com/android/inputmethod/latin/WordComposerTests.java +++ b/tests/src/com/android/inputmethod/latin/WordComposerTests.java @@ -57,14 +57,14 @@ public class WordComposerTests extends AndroidTestCase { assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); // Check the previous word is still there - assertEquals(PREVWORD, wc.getPreviousWordForSuggestion()); + assertEquals(PREVWORD, wc.getPrevWordsInfoForSuggestion().mPrevWord); // Move the cursor past the end of the word assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(15)); // Do what LatinIME does when the cursor is moved outside of the word, // and check the behavior is correct. wc.reset(); - assertNull(wc.getPreviousWordForSuggestion()); + assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord); // \uD861\uDED7 is 𨛗, a character outside the BMP final String STR_WITH_SUPPLEMENTARY_CHAR = "abcde\uD861\uDED7fgh"; @@ -83,37 +83,37 @@ public class WordComposerTests extends AndroidTestCase { assertTrue(wc.isCursorFrontOrMiddleOfComposingWord()); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(1)); assertFalse(wc.isCursorFrontOrMiddleOfComposingWord()); - assertNull(wc.getPreviousWordForSuggestion()); + assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, STR_WITHIN_BMP); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - assertEquals(STR_WITHIN_BMP, wc.getPreviousWordForSuggestion()); + assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, STR_WITH_SUPPLEMENTARY_CHAR); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(7)); - assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWordForSuggestion()); + assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, STR_WITHIN_BMP); wc.setCursorPositionWithinWord(3); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-3)); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-1)); - assertEquals(STR_WITHIN_BMP, wc.getPreviousWordForSuggestion()); + assertEquals(STR_WITHIN_BMP, wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, null /* previousWord */); wc.setCursorPositionWithinWord(3); assertFalse(wc.moveCursorByAndReturnIfInsideComposingWord(-9)); - assertNull(wc.getPreviousWordForSuggestion()); + assertNull(wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, STR_WITH_SUPPLEMENTARY_CHAR); assertTrue(wc.moveCursorByAndReturnIfInsideComposingWord(-10)); - assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPreviousWordForSuggestion()); + assertEquals(STR_WITH_SUPPLEMENTARY_CHAR, wc.getPrevWordsInfoForSuggestion().mPrevWord); wc.setComposingWord(CODEPOINTS_WITH_SUPPLEMENTARY_CHAR, COORDINATES_WITH_SUPPLEMENTARY_CHAR, null /* previousWord */); diff --git a/tools/dicttool/Android.mk b/tools/dicttool/Android.mk index e12d7e0b5..10104cf33 100644 --- a/tools/dicttool/Android.mk +++ b/tools/dicttool/Android.mk @@ -44,6 +44,7 @@ LATINIME_SRC_FILES_FOR_DICTTOOL := \ latin/InputPointers.java \ latin/LastComposedWord.java \ latin/LatinImeLogger.java \ + latin/PrevWordsInfo.java \ latin/SuggestedWords.java \ latin/WordComposer.java \ latin/settings/NativeSuggestOptions.java \