nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote branch 'goog/master'

Browse source 
Conflicts:
	java/res/xml/prefs.xml
	java/src/com/android/inputmethod/latin/BinaryDictionary.java
	java/src/com/android/inputmethod/latin/Dictionary.java
	java/src/com/android/inputmethod/latin/ExpandableDictionary.java
	java/src/com/android/inputmethod/latin/LatinIME.java
	java/src/com/android/inputmethod/latin/Suggest.java
	tests/src/com/android/inputmethod/latin/tests/SuggestTests.java
This commit is contained in:
satok 2010-07-16 13:02:45 +09:00
commit b9c57e6540
18 changed files with 688 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
java/Android.mk
View file

@ -18,7 +18,7 @@ LOCAL_STATIC_JAVA_LIBRARIES := android-common
# com.google.android.inputmethod.latin in the LatinIME sandbox. # com.google.android.inputmethod.latin in the LatinIME sandbox.
LOCAL_AAPT_FLAGS := --custom-package com.android.inputmethod.latin LOCAL_AAPT_FLAGS := --custom-package com.android.inputmethod.latin
LOCAL_SDK_VERSION := current LOCAL_SDK_VERSION := 8
LOCAL_PROGUARD_FLAGS := -include $(LOCAL_PATH)/proguard.flags LOCAL_PROGUARD_FLAGS := -include $(LOCAL_PATH)/proguard.flags

5
java/res/values/strings.xml
View file

@ -86,6 +86,11 @@
<!-- Description for auto completion --> <!-- Description for auto completion -->
<string name="auto_complete_summary">Spacebar and punctuation automatically insert highlighted word</string> <string name="auto_complete_summary">Spacebar and punctuation automatically insert highlighted word</string>
<!-- Option to enable bigram completion -->
<string name="bigram_suggestion">Bigram Suggestions</string>
<!-- Description for auto completion -->
<string name="bigram_suggestion_summary">Use previous word to improve suggestion</string>
<!-- Array of prediction modes --> <!-- Array of prediction modes -->
<string-array name="prediction_modes"> <string-array name="prediction_modes">
<item>None</item> <item>None</item>

14
java/res/xml/prefs.xml
View file

@ -98,10 +98,18 @@
android:defaultValue="@bool/enable_autocorrect" android:defaultValue="@bool/enable_autocorrect"
android:dependency="show_suggestions" android:dependency="show_suggestions"
/> />
</PreferenceCategory>
<CheckBoxPreference <CheckBoxPreference
android:key="bigram_suggestion"
android:title="@string/bigram_suggestion"
android:summary="@string/bigram_suggestion_summary"
android:persistent="true"
android:defaultValue="true"
android:dependency="auto_complete"
/>
</PreferenceCategory>
<CheckBoxPreference
android:key="debug_mode" android:key="debug_mode"
android:title="@string/prefs_debug_mode" android:title="@string/prefs_debug_mode"
android:persistent="true" android:persistent="true"

34
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -32,9 +32,9 @@ import android.util.Log;
public class BinaryDictionary extends Dictionary { public class BinaryDictionary extends Dictionary {
private static final String TAG = "BinaryDictionary"; private static final String TAG = "BinaryDictionary";
public static final int MAX_WORD_LENGTH = 48;
private static final int MAX_ALTERNATIVES = 16; private static final int MAX_ALTERNATIVES = 16;
private static final int MAX_WORDS = 16; private static final int MAX_WORDS = 16;
private static final int MAX_BIGRAMS = 255; // TODO Probably don't need all 255
private static final int TYPED_LETTER_MULTIPLIER = 2; private static final int TYPED_LETTER_MULTIPLIER = 2;
private static final boolean ENABLE_MISSED_CHARACTERS = true; private static final boolean ENABLE_MISSED_CHARACTERS = true;
@ -44,7 +44,9 @@ public class BinaryDictionary extends Dictionary {
private int mDictLength; private int mDictLength;
private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES]; private int[] mInputCodes = new int[MAX_WORD_LENGTH * MAX_ALTERNATIVES];
private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS]; private char[] mOutputChars = new char[MAX_WORD_LENGTH * MAX_WORDS];
private char[] mOutputChars_bigrams = new char[MAX_WORD_LENGTH * MAX_BIGRAMS];
private int[] mFrequencies = new int[MAX_WORDS]; private int[] mFrequencies = new int[MAX_WORDS];
private int[] mFrequencies_bigrams = new int[MAX_BIGRAMS];
// Keep a reference to the native dict direct buffer in Java to avoid // Keep a reference to the native dict direct buffer in Java to avoid
// unexpected deallocation of the direct buffer. // unexpected deallocation of the direct buffer.
private ByteBuffer mNativeDictDirectBuffer; private ByteBuffer mNativeDictDirectBuffer;
@ -72,7 +74,7 @@ public class BinaryDictionary extends Dictionary {
/** /**
* Create a dictionary from a byte buffer. This is used for testing. * Create a dictionary from a byte buffer. This is used for testing.
* @param context application context for reading resources * @param context application context for reading resources
* @param resId the resource containing the raw binary dictionary * @param byteBuffer a ByteBuffer containing the binary dictionary
*/ */
public BinaryDictionary(Context context, ByteBuffer byteBuffer, int dicTypeId) { public BinaryDictionary(Context context, ByteBuffer byteBuffer, int dicTypeId) {
if (byteBuffer != null) { if (byteBuffer != null) {
@ -97,6 +99,8 @@ public class BinaryDictionary extends Dictionary {
char[] outputChars, int[] frequencies, char[] outputChars, int[] frequencies,
int maxWordLength, int maxWords, int maxAlternatives, int skipPos, int maxWordLength, int maxWords, int maxAlternatives, int skipPos,
int[] nextLettersFrequencies, int nextLettersSize); int[] nextLettersFrequencies, int nextLettersSize);
private native int getBigramsNative(int nativeData, char[] prevWord, int prevWordLength,
char[] outputChars, int[] frequencies, int maxWordLength, int maxBigrams);
private final void loadDictionary(Context context, int resId) { private final void loadDictionary(Context context, int resId) {
InputStream is = context.getResources().openRawResource(resId); InputStream is = context.getResources().openRawResource(resId);
@ -123,6 +127,30 @@ public class BinaryDictionary extends Dictionary {
} }
} }
@Override
public void getBigrams(final WordComposer composer, final CharSequence previousWord,
final WordCallback callback, int[] nextLettersFrequencies) {
char[] chars = previousWord.toString().toCharArray();
Arrays.fill(mOutputChars_bigrams, (char) 0);
Arrays.fill(mFrequencies_bigrams, 0);
int count = getBigramsNative(mNativeDict, chars, chars.length, mOutputChars_bigrams,
mFrequencies_bigrams, MAX_WORD_LENGTH, MAX_BIGRAMS);
for (int j = 0; j < count; j++) {
if (mFrequencies_bigrams[j] < 1) break;
int start = j * MAX_WORD_LENGTH;
int len = 0;
while (mOutputChars_bigrams[start + len] != 0) {
len++;
}
if (len > 0) {
callback.addWord(mOutputChars_bigrams, start, len, mFrequencies_bigrams[j],
DataType.BIGRAM);
}
}
}
@Override @Override
public void getWords(final WordComposer codes, final WordCallback callback, public void getWords(final WordComposer codes, final WordCallback callback,
int[] nextLettersFrequencies) { int[] nextLettersFrequencies) {
@ -168,7 +196,7 @@ public class BinaryDictionary extends Dictionary {
len++; len++;
} }
if (len > 0) { if (len > 0) {
callback.addWord(mOutputChars, start, len, mFrequencies[j], mDicTypeId); callback.addWord(mOutputChars, start, len, mFrequencies[j], mDicTypeId, DataType.UNIGRAM);
} }
} }
} }

29
java/src/com/android/inputmethod/latin/Dictionary.java
View file

@ -21,7 +21,9 @@ package com.android.inputmethod.latin;
* strokes. * strokes.
*/ */
abstract public class Dictionary { abstract public class Dictionary {
protected static final int MAX_WORD_LENGTH = 48;
/** /**
* Whether or not to replicate the typed word in the suggested list, even if it's valid. * Whether or not to replicate the typed word in the suggested list, even if it's valid.
*/ */
@ -31,7 +33,11 @@ abstract public class Dictionary {
* The weight to give to a word if it's length is the same as the number of typed characters. * The weight to give to a word if it's length is the same as the number of typed characters.
*/ */
protected static final int FULL_WORD_FREQ_MULTIPLIER = 2; protected static final int FULL_WORD_FREQ_MULTIPLIER = 2;
public static enum DataType {
UNIGRAM, BIGRAM
}
/** /**
* Interface to be implemented by classes requesting words to be fetched from the dictionary. * Interface to be implemented by classes requesting words to be fetched from the dictionary.
* @see #getWords(WordComposer, WordCallback) * @see #getWords(WordComposer, WordCallback)
@ -46,9 +52,11 @@ abstract public class Dictionary {
* @param frequency the frequency of occurence. This is normalized between 1 and 255, but * @param frequency the frequency of occurence. This is normalized between 1 and 255, but
* can exceed those limits * can exceed those limits
* @param dicTypeId of the dictionary where word was from * @param dicTypeId of the dictionary where word was from
* @param dataType tells type of this data
* @return true if the word was added, false if no more words are required * @return true if the word was added, false if no more words are required
*/ */
boolean addWord(char[] word, int wordOffset, int wordLength, int frequency, int dicTypeId); boolean addWord(char[] word, int wordOffset, int wordLength, int frequency, int dicTypeId,
DataType dataType);
} }
/** /**
@ -65,6 +73,21 @@ abstract public class Dictionary {
abstract public void getWords(final WordComposer composer, final WordCallback callback, abstract public void getWords(final WordComposer composer, final WordCallback callback,
int[] nextLettersFrequencies); int[] nextLettersFrequencies);
/**
* Searches for pairs in the bigram dictionary that matches the previous word and all the
* possible words following are added through the callback object.
* @param composer the key sequence to match
* @param callback the callback object to send possible word following previous word
* @param nextLettersFrequencies array of frequencies of next letters that could follow the
* word so far. For instance, "bracke" can be followed by "t", so array['t'] will have
* a non-zero value on returning from this method.
* Pass in null if you don't want the dictionary to look up next letters.
*/
public void getBigrams(final WordComposer composer, final CharSequence previousWord,
final WordCallback callback, int[] nextLettersFrequencies) {
// empty base implementation
}
/** /**
* Checks if the given word occurs in the dictionary * Checks if the given word occurs in the dictionary
* @param word the word to search for. The search should be case-insensitive. * @param word the word to search for. The search should be case-insensitive.

16
java/src/com/android/inputmethod/latin/EditingUtil.java
View file

@ -16,6 +16,8 @@
package com.android.inputmethod.latin; package com.android.inputmethod.latin;
import java.util.regex.Pattern;
import android.view.inputmethod.ExtractedText; import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputConnection;
@ -24,6 +26,11 @@ import android.view.inputmethod.InputConnection;
* Utility methods to deal with editing text through an InputConnection. * Utility methods to deal with editing text through an InputConnection.
*/ */
public class EditingUtil { public class EditingUtil {
/**
* Number of characters we want to look back in order to identify the previous word
*/
public static final int LOOKBACK_CHARACTER_NUM = 15;
private EditingUtil() {}; private EditingUtil() {};
/** /**
@ -175,4 +182,13 @@ public class EditingUtil {
private static boolean isWhitespace(int code, String whitespace) { private static boolean isWhitespace(int code, String whitespace) {
return whitespace.contains(String.valueOf((char) code)); return whitespace.contains(String.valueOf((char) code));
} }
private static final Pattern spaceRegex = Pattern.compile("\\s+");
public static CharSequence getPreviousWord(InputConnection connection) {
//TODO: Should fix this. This could be slow!
CharSequence prev = connection.getTextBeforeCursor(LOOKBACK_CHARACTER_NUM, 0);
String[] w = spaceRegex.split(prev);
return (w.length >= 2) ? w[w.length-2] : null;
}
} }

6
java/src/com/android/inputmethod/latin/ExpandableDictionary.java
View file

@ -266,7 +266,8 @@ public class ExpandableDictionary extends Dictionary {
if (completion) { if (completion) {
word[depth] = c; word[depth] = c;
if (terminal) { if (terminal) {
if (!callback.addWord(word, 0, depth + 1, freq * snr, mDicTypeId)) { if (!callback.addWord(word, 0, depth + 1, freq * snr, mDicTypeId,
DataType.UNIGRAM)) {
return; return;
} }
// Add to frequency of next letters for predictive correction // Add to frequency of next letters for predictive correction
@ -304,7 +305,8 @@ public class ExpandableDictionary extends Dictionary {
|| !same(word, depth + 1, codes.getTypedWord())) { || !same(word, depth + 1, codes.getTypedWord())) {
int finalFreq = freq * snr * addedAttenuation; int finalFreq = freq * snr * addedAttenuation;
if (skipPos < 0) finalFreq *= FULL_WORD_FREQ_MULTIPLIER; if (skipPos < 0) finalFreq *= FULL_WORD_FREQ_MULTIPLIER;
callback.addWord(word, 0, depth + 1, finalFreq, mDicTypeId); callback.addWord(word, 0, depth + 1, finalFreq, mDicTypeId,
DataType.UNIGRAM);
} }
} }
if (children != null) { if (children != null) {

57
java/src/com/android/inputmethod/latin/LatinIME.java
View file

@ -48,8 +48,8 @@ import android.view.HapticFeedbackConstants;
import android.view.KeyEvent; import android.view.KeyEvent;
import android.view.LayoutInflater; import android.view.LayoutInflater;
import android.view.View; import android.view.View;
import android.view.ViewParent;
import android.view.ViewGroup; import android.view.ViewGroup;
import android.view.ViewParent;
import android.view.Window; import android.view.Window;
import android.view.WindowManager; import android.view.WindowManager;
import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.CompletionInfo;
@ -88,6 +88,7 @@ public class LatinIME extends InputMethodService
private static final String PREF_QUICK_FIXES = "quick_fixes"; private static final String PREF_QUICK_FIXES = "quick_fixes";
private static final String PREF_SHOW_SUGGESTIONS = "show_suggestions"; private static final String PREF_SHOW_SUGGESTIONS = "show_suggestions";
private static final String PREF_AUTO_COMPLETE = "auto_complete"; private static final String PREF_AUTO_COMPLETE = "auto_complete";
private static final String PREF_BIGRAM_SUGGESTIONS = "bigram_suggestion";
private static final String PREF_VOICE_MODE = "voice_mode"; private static final String PREF_VOICE_MODE = "voice_mode";
// Whether or not the user has used voice input before (and thus, whether to show the // Whether or not the user has used voice input before (and thus, whether to show the
@ -186,6 +187,7 @@ public class LatinIME extends InputMethodService
private boolean mAutoSpace; private boolean mAutoSpace;
private boolean mJustAddedAutoSpace; private boolean mJustAddedAutoSpace;
private boolean mAutoCorrectEnabled; private boolean mAutoCorrectEnabled;
private boolean mBigramSuggestionEnabled;
private boolean mAutoCorrectOn; private boolean mAutoCorrectOn;
private boolean mCapsLock; private boolean mCapsLock;
private boolean mPasswordText; private boolean mPasswordText;
@ -713,12 +715,14 @@ public class LatinIME extends InputMethodService
// TODO: Uncomment this block when we enable re-editing feature // TODO: Uncomment this block when we enable re-editing feature
// If a word is selected // If a word is selected
/*if ((candidatesStart == candidatesEnd || newSelStart != oldSelStart) /*if (isPredictionOn() && mJustRevertedSeparator == null
&& (candidatesStart == candidatesEnd || newSelStart != oldSelStart)
&& (newSelStart < newSelEnd - 1 || (!mPredicting)) && (newSelStart < newSelEnd - 1 || (!mPredicting))
&& !mVoiceInputHighlighted) { && !mVoiceInputHighlighted) {
abortCorrection(false);
if (isCursorTouchingWord() || mLastSelectionStart < mLastSelectionEnd) { if (isCursorTouchingWord() || mLastSelectionStart < mLastSelectionEnd) {
postUpdateOldSuggestions(); postUpdateOldSuggestions();
} else {
abortCorrection(false);
} }
}*/ }*/
} }
@ -1113,6 +1117,8 @@ public class LatinIME extends InputMethodService
InputConnection ic = getCurrentInputConnection(); InputConnection ic = getCurrentInputConnection();
if (ic == null) return; if (ic == null) return;
ic.beginBatchEdit();
if (mAfterVoiceInput) { if (mAfterVoiceInput) {
// Don't log delete if the user is pressing delete at // Don't log delete if the user is pressing delete at
// the beginning of the text box (hence not deleting anything) // the beginning of the text box (hence not deleting anything)
@ -1145,6 +1151,7 @@ public class LatinIME extends InputMethodService
TextEntryState.backspace(); TextEntryState.backspace();
if (TextEntryState.getState() == TextEntryState.STATE_UNDO_COMMIT) { if (TextEntryState.getState() == TextEntryState.STATE_UNDO_COMMIT) {
revertLastWord(deleteChar); revertLastWord(deleteChar);
ic.endBatchEdit();
return; return;
} else if (mEnteredText != null && sameAsTextBeforeCursor(ic, mEnteredText)) { } else if (mEnteredText != null && sameAsTextBeforeCursor(ic, mEnteredText)) {
ic.deleteSurroundingText(mEnteredText.length(), 0); ic.deleteSurroundingText(mEnteredText.length(), 0);
@ -1155,6 +1162,7 @@ public class LatinIME extends InputMethodService
} }
} }
mJustRevertedSeparator = null; mJustRevertedSeparator = null;
ic.endBatchEdit();
} }
private void handleShift() { private void handleShift() {
@ -1312,9 +1320,10 @@ public class LatinIME extends InputMethodService
mWord.reset(); mWord.reset();
return; return;
} }
TypedWordAlternatives entry = new TypedWordAlternatives(result, mWord); // Make a copy of the CharSequence, since it is/could be a mutable CharSequence
// Create a new WordComposer as the old one is being saved for later use final String resultCopy = result.toString();
mWord = new WordComposer(mWord); TypedWordAlternatives entry = new TypedWordAlternatives(resultCopy,
new WordComposer(mWord));
mWordHistory.add(entry); mWordHistory.add(entry);
} }
@ -1569,8 +1578,7 @@ public class LatinIME extends InputMethodService
} }
private List<CharSequence> getTypedSuggestions(WordComposer word) { private List<CharSequence> getTypedSuggestions(WordComposer word) {
List<CharSequence> stringList = mSuggest.getSuggestions( List<CharSequence> stringList = mSuggest.getSuggestions(mKeyboardSwitcher.getInputView(), word, false, null);
mKeyboardSwitcher.getInputView(), word, false);
return stringList; return stringList;
} }
@ -1581,8 +1589,14 @@ public class LatinIME extends InputMethodService
} }
private void showSuggestions(WordComposer word) { private void showSuggestions(WordComposer word) {
List<CharSequence> stringList = mSuggest.getSuggestions( //long startTime = System.currentTimeMillis(); // TIME MEASUREMENT!
mKeyboardSwitcher.getInputView(), word, false); // TODO Maybe need better way of retrieving previous word
CharSequence prevWord = EditingUtil.getPreviousWord(getCurrentInputConnection());
List<CharSequence> stringList = mSuggest.getSuggestions(mKeyboardSwitcher.getInputView(), word, false,
prevWord);
//long stopTime = System.currentTimeMillis(); // TIME MEASUREMENT!
//Log.d("LatinIME","Suggest Total Time - " + (stopTime - startTime));
int[] nextLettersFrequencies = mSuggest.getNextLettersFrequencies(); int[] nextLettersFrequencies = mSuggest.getNextLettersFrequencies();
((LatinKeyboard) mKeyboardSwitcher.getInputView().getKeyboard()).setPreferredLetters( ((LatinKeyboard) mKeyboardSwitcher.getInputView().getKeyboard()).setPreferredLetters(
@ -1699,7 +1713,8 @@ public class LatinIME extends InputMethodService
// Fool the state watcher so that a subsequent backspace will not do a revert // Fool the state watcher so that a subsequent backspace will not do a revert
TextEntryState.typedCharacter((char) KEYCODE_SPACE, true); TextEntryState.typedCharacter((char) KEYCODE_SPACE, true);
if (index == 0 && mCorrectionMode > 0 && !mSuggest.isValidWord(suggestion)) { if (index == 0 && mCorrectionMode > 0 && !mSuggest.isValidWord(suggestion)
&& !mSuggest.isValidWord(suggestion.toString().toLowerCase())) {
mCandidateView.showAddToDictionaryHint(suggestion); mCandidateView.showAddToDictionaryHint(suggestion);
} }
if (ic != null) { if (ic != null) {
@ -1713,9 +1728,9 @@ public class LatinIME extends InputMethodService
InputConnection ic = getCurrentInputConnection(); InputConnection ic = getCurrentInputConnection();
EditingUtil.Range range = new EditingUtil.Range(); EditingUtil.Range range = new EditingUtil.Range();
String wordToBeReplaced = EditingUtil.getWordAtCursor(getCurrentInputConnection(), String wordToBeReplaced = EditingUtil.getWordAtCursor(getCurrentInputConnection(),
mWordSeparators, range).trim(); mWordSeparators, range);
if (!mWordToSuggestions.containsKey(wordToBeReplaced)) { if (!mWordToSuggestions.containsKey(wordToBeReplaced)) {
wordToBeReplaced = wordToBeReplaced.toLowerCase(); wordToBeReplaced = wordToBeReplaced.toLowerCase();
} }
if (mWordToSuggestions.containsKey(wordToBeReplaced)) { if (mWordToSuggestions.containsKey(wordToBeReplaced)) {
List<CharSequence> suggestions = mWordToSuggestions.get(wordToBeReplaced); List<CharSequence> suggestions = mWordToSuggestions.get(wordToBeReplaced);
@ -1743,9 +1758,6 @@ public class LatinIME extends InputMethodService
InputConnection ic = getCurrentInputConnection(); InputConnection ic = getCurrentInputConnection();
if (ic != null) { if (ic != null) {
rememberReplacedWord(suggestion); rememberReplacedWord(suggestion);
if (mSuggestionShouldReplaceCurrentWord) {
EditingUtil.deleteWordAtCursor(ic, getWordSeparators());
}
if (!VoiceInput.DELETE_SYMBOL.equals(suggestion)) { if (!VoiceInput.DELETE_SYMBOL.equals(suggestion)) {
ic.commitText(suggestion, 1); ic.commitText(suggestion, 1);
} }
@ -1772,9 +1784,8 @@ public class LatinIME extends InputMethodService
} }
if (!mPredicting && isCursorTouchingWord()) { if (!mPredicting && isCursorTouchingWord()) {
EditingUtil.Range range = new EditingUtil.Range(); EditingUtil.Range range = new EditingUtil.Range();
CharSequence touching = CharSequence touching = EditingUtil.getWordAtCursor(getCurrentInputConnection(),
EditingUtil.getWordAtCursor(getCurrentInputConnection(), mWordSeparators, mWordSeparators, range);
range);
if (touching != null && touching.length() > 1) { if (touching != null && touching.length() > 1) {
if (mWordSeparators.indexOf(touching.charAt(touching.length() - 1)) > 0) { if (mWordSeparators.indexOf(touching.charAt(touching.length() - 1)) > 0) {
touching = touching.toString().substring(0, touching.length() - 1); touching = touching.toString().substring(0, touching.length() - 1);
@ -1835,7 +1846,7 @@ public class LatinIME extends InputMethodService
foundWord); foundWord);
showCorrections(alternatives); showCorrections(alternatives);
if (foundWord != null) { if (foundWord != null) {
mWord = foundWord; mWord = new WordComposer(foundWord);
} else { } else {
mWord.reset(); mWord.reset();
} }
@ -1868,6 +1879,7 @@ public class LatinIME extends InputMethodService
private void underlineWord(CharSequence word, int left, int right) { private void underlineWord(CharSequence word, int left, int right) {
InputConnection ic = getCurrentInputConnection(); InputConnection ic = getCurrentInputConnection();
if (ic == null) return; if (ic == null) return;
ic.finishComposingText();
ic.deleteSurroundingText(left, right); ic.deleteSurroundingText(left, right);
ic.setComposingText(word, 1); ic.setComposingText(word, 1);
ic.setSelection(mLastSelectionStart, mLastSelectionStart); ic.setSelection(mLastSelectionStart, mLastSelectionStart);
@ -1912,7 +1924,6 @@ public class LatinIME extends InputMethodService
if (!mPredicting && length > 0) { if (!mPredicting && length > 0) {
final InputConnection ic = getCurrentInputConnection(); final InputConnection ic = getCurrentInputConnection();
mPredicting = true; mPredicting = true;
ic.beginBatchEdit();
mJustRevertedSeparator = ic.getTextBeforeCursor(1, 0); mJustRevertedSeparator = ic.getTextBeforeCursor(1, 0);
if (deleteChar) ic.deleteSurroundingText(1, 0); if (deleteChar) ic.deleteSurroundingText(1, 0);
int toDelete = mCommittedLength; int toDelete = mCommittedLength;
@ -1924,7 +1935,6 @@ public class LatinIME extends InputMethodService
ic.deleteSurroundingText(toDelete, 0); ic.deleteSurroundingText(toDelete, 0);
ic.setComposingText(mComposing, 1); ic.setComposingText(mComposing, 1);
TextEntryState.backspace(); TextEntryState.backspace();
ic.endBatchEdit();
postUpdateSuggestions(); postUpdateSuggestions();
} else { } else {
sendDownUpKeyEvents(KeyEvent.KEYCODE_DEL); sendDownUpKeyEvents(KeyEvent.KEYCODE_DEL);
@ -2139,6 +2149,8 @@ public class LatinIME extends InputMethodService
mCorrectionMode = (mAutoCorrectOn && mAutoCorrectEnabled) mCorrectionMode = (mAutoCorrectOn && mAutoCorrectEnabled)
? Suggest.CORRECTION_FULL ? Suggest.CORRECTION_FULL
: (mAutoCorrectOn ? Suggest.CORRECTION_BASIC : Suggest.CORRECTION_NONE); : (mAutoCorrectOn ? Suggest.CORRECTION_BASIC : Suggest.CORRECTION_NONE);
mCorrectionMode = (mBigramSuggestionEnabled && mAutoCorrectOn && mAutoCorrectEnabled)
? Suggest.CORRECTION_FULL_BIGRAM : mCorrectionMode;
if (mSuggest != null) { if (mSuggest != null) {
mSuggest.setCorrectionMode(mCorrectionMode); mSuggest.setCorrectionMode(mCorrectionMode);
} }
@ -2205,6 +2217,7 @@ public class LatinIME extends InputMethodService
} }
mAutoCorrectEnabled = sp.getBoolean(PREF_AUTO_COMPLETE, mAutoCorrectEnabled = sp.getBoolean(PREF_AUTO_COMPLETE,
mResources.getBoolean(R.bool.enable_autocorrect)) & mShowSuggestions; mResources.getBoolean(R.bool.enable_autocorrect)) & mShowSuggestions;
mBigramSuggestionEnabled = sp.getBoolean(PREF_BIGRAM_SUGGESTIONS, true) & mShowSuggestions;
updateCorrectionMode(); updateCorrectionMode();
updateAutoTextEnabled(mResources.getConfiguration().locale); updateAutoTextEnabled(mResources.getConfiguration().locale);
mLanguageSwitcher.loadLocales(sp); mLanguageSwitcher.loadLocales(sp);

166
java/src/com/android/inputmethod/latin/Suggest.java
View file

@ -37,6 +37,21 @@ public class Suggest implements Dictionary.WordCallback {
public static final int CORRECTION_NONE = 0; public static final int CORRECTION_NONE = 0;
public static final int CORRECTION_BASIC = 1; public static final int CORRECTION_BASIC = 1;
public static final int CORRECTION_FULL = 2; public static final int CORRECTION_FULL = 2;
public static final int CORRECTION_FULL_BIGRAM = 3;
/**
* Words that appear in both bigram and unigram data gets multiplier ranging from
* BIGRAM_MULTIPLIER_MIN to BIGRAM_MULTIPLIER_MAX depending on the frequency score from
* bigram data.
*/
public static final double BIGRAM_MULTIPLIER_MIN = 1.2;
public static final double BIGRAM_MULTIPLIER_MAX = 1.5;
/**
* Maximum possible bigram frequency. Will depend on how many bits are being used in data
* structure. Maximum bigram freqeuncy will get the BIGRAM_MULTIPLIER_MAX as the multiplier.
*/
public static final int MAXIMUM_BIGRAM_FREQUENCY = 127;
public static final int DIC_USER_TYPED = 0; public static final int DIC_USER_TYPED = 0;
public static final int DIC_MAIN = 1; public static final int DIC_MAIN = 1;
@ -57,10 +72,13 @@ public class Suggest implements Dictionary.WordCallback {
private Dictionary mContactsDictionary; private Dictionary mContactsDictionary;
private int mPrefMaxSuggestions = 12; private int mPrefMaxSuggestions = 12;
private int mPrefMaxBigrams = 255;
private boolean mAutoTextEnabled; private boolean mAutoTextEnabled;
private int[] mPriorities = new int[mPrefMaxSuggestions]; private int[] mPriorities = new int[mPrefMaxSuggestions];
private int[] mBigramPriorities = new int[mPrefMaxBigrams];
// Handle predictive correction for only the first 1280 characters for performance reasons // Handle predictive correction for only the first 1280 characters for performance reasons
// If we support scripts that need latin characters beyond that, we should probably use some // If we support scripts that need latin characters beyond that, we should probably use some
// kind of a sparse array or language specific list with a mapping lookup table. // kind of a sparse array or language specific list with a mapping lookup table.
@ -68,6 +86,7 @@ public class Suggest implements Dictionary.WordCallback {
// latin characters. // latin characters.
private int[] mNextLettersFrequencies = new int[1280]; private int[] mNextLettersFrequencies = new int[1280];
private ArrayList<CharSequence> mSuggestions = new ArrayList<CharSequence>(); private ArrayList<CharSequence> mSuggestions = new ArrayList<CharSequence>();
private ArrayList<CharSequence> mBigramSuggestions = new ArrayList<CharSequence>();
private ArrayList<CharSequence> mStringPool = new ArrayList<CharSequence>(); private ArrayList<CharSequence> mStringPool = new ArrayList<CharSequence>();
private boolean mHaveCorrection; private boolean mHaveCorrection;
private CharSequence mOriginalWord; private CharSequence mOriginalWord;
@ -88,7 +107,7 @@ public class Suggest implements Dictionary.WordCallback {
private void initPool() { private void initPool() {
for (int i = 0; i < mPrefMaxSuggestions; i++) { for (int i = 0; i < mPrefMaxSuggestions; i++) {
StringBuilder sb = new StringBuilder(32); StringBuilder sb = new StringBuilder(Dictionary.MAX_WORD_LENGTH);
mStringPool.add(sb); mStringPool.add(sb);
} }
} }
@ -140,9 +159,10 @@ public class Suggest implements Dictionary.WordCallback {
} }
mPrefMaxSuggestions = maxSuggestions; mPrefMaxSuggestions = maxSuggestions;
mPriorities = new int[mPrefMaxSuggestions]; mPriorities = new int[mPrefMaxSuggestions];
collectGarbage(); mBigramPriorities = new int[mPrefMaxBigrams];
collectGarbage(mSuggestions, mPrefMaxSuggestions);
while (mStringPool.size() < mPrefMaxSuggestions) { while (mStringPool.size() < mPrefMaxSuggestions) {
StringBuilder sb = new StringBuilder(32); StringBuilder sb = new StringBuilder(Dictionary.MAX_WORD_LENGTH);
mStringPool.add(sb); mStringPool.add(sb);
} }
} }
@ -177,18 +197,17 @@ public class Suggest implements Dictionary.WordCallback {
/** /**
* Returns a list of words that match the list of character codes passed in. * Returns a list of words that match the list of character codes passed in.
* This list will be overwritten the next time this function is called. * This list will be overwritten the next time this function is called.
* @param a view for retrieving the context for AutoText * @param view a view for retrieving the context for AutoText
* @param codes the list of codes. Each list item contains an array of character codes * @param wordComposer contains what is currently being typed
* in order of probability where the character at index 0 in the array has the highest * @param prevWordForBigram previous word (used only for bigram)
* probability.
* @return list of suggestions. * @return list of suggestions.
*/ */
public List<CharSequence> getSuggestions(View view, WordComposer wordComposer, public List<CharSequence> getSuggestions(View view, WordComposer wordComposer,
boolean includeTypedWordIfValid) { boolean includeTypedWordIfValid, CharSequence prevWordForBigram) {
LatinImeLogger.onStartSuggestion(); LatinImeLogger.onStartSuggestion();
mHaveCorrection = false; mHaveCorrection = false;
mCapitalize = wordComposer.isCapitalized(); mCapitalize = wordComposer.isCapitalized();
collectGarbage(); collectGarbage(mSuggestions, mPrefMaxSuggestions);
Arrays.fill(mPriorities, 0); Arrays.fill(mPriorities, 0);
Arrays.fill(mNextLettersFrequencies, 0); Arrays.fill(mNextLettersFrequencies, 0);
@ -203,7 +222,38 @@ public class Suggest implements Dictionary.WordCallback {
} }
// Search the dictionary only if there are at least 2 characters // Search the dictionary only if there are at least 2 characters
if (wordComposer.size() > 1) { if (wordComposer.size() == 1 && (mCorrectionMode == CORRECTION_FULL_BIGRAM
|| mCorrectionMode == CORRECTION_BASIC)) {
// At first character, just get the bigrams
Arrays.fill(mBigramPriorities, 0);
collectGarbage(mBigramSuggestions, mPrefMaxBigrams);
if (!TextUtils.isEmpty(prevWordForBigram)) {
CharSequence lowerPrevWord = prevWordForBigram.toString().toLowerCase();
if (mMainDict.isValidWord(lowerPrevWord)) {
prevWordForBigram = lowerPrevWord;
}
mMainDict.getBigrams(wordComposer, prevWordForBigram, this,
mNextLettersFrequencies);
char currentChar = wordComposer.getTypedWord().charAt(0);
int count = 0;
int bigramSuggestionSize = mBigramSuggestions.size();
for (int i = 0; i < bigramSuggestionSize; i++) {
if (mBigramSuggestions.get(i).charAt(0) == currentChar) {
int poolSize = mStringPool.size();
StringBuilder sb = poolSize > 0 ?
(StringBuilder) mStringPool.remove(poolSize - 1)
: new StringBuilder(Dictionary.MAX_WORD_LENGTH);
sb.setLength(0);
sb.append(mBigramSuggestions.get(i));
mSuggestions.add(count++, sb);
if (count > mPrefMaxSuggestions) break;
}
}
}
} else if (wordComposer.size() > 1) {
// Search the dictionary only if there are at least 2 characters
if (mUserDictionary != null || mContactsDictionary != null) { if (mUserDictionary != null || mContactsDictionary != null) {
if (mUserDictionary != null) { if (mUserDictionary != null) {
mUserDictionary.getWords(wordComposer, this, mNextLettersFrequencies); mUserDictionary.getWords(wordComposer, this, mNextLettersFrequencies);
@ -213,21 +263,26 @@ public class Suggest implements Dictionary.WordCallback {
} }
if (mSuggestions.size() > 0 && isValidWord(mOriginalWord) if (mSuggestions.size() > 0 && isValidWord(mOriginalWord)
&& mCorrectionMode == CORRECTION_FULL) { && (mCorrectionMode == CORRECTION_FULL
|| mCorrectionMode == CORRECTION_FULL_BIGRAM)) {
mHaveCorrection = true; mHaveCorrection = true;
} }
} }
mMainDict.getWords(wordComposer, this, mNextLettersFrequencies); mMainDict.getWords(wordComposer, this, mNextLettersFrequencies);
if (mCorrectionMode == CORRECTION_FULL && mSuggestions.size() > 0) { if ((mCorrectionMode == CORRECTION_FULL || mCorrectionMode == CORRECTION_FULL_BIGRAM)
&& mSuggestions.size() > 0) {
mHaveCorrection = true; mHaveCorrection = true;
} }
} }
if (mOriginalWord != null) { if (mOriginalWord != null) {
mSuggestions.add(0, mOriginalWord.toString()); mSuggestions.add(0, mOriginalWord.toString());
} }
// Check if the first suggestion has a minimum number of characters in common // Check if the first suggestion has a minimum number of characters in common
if (mCorrectionMode == CORRECTION_FULL && mSuggestions.size() > 1) { if (wordComposer.size() > 1 && mSuggestions.size() > 1
&& (mCorrectionMode == CORRECTION_FULL
|| mCorrectionMode == CORRECTION_FULL_BIGRAM)) {
if (!haveSufficientCommonality(mLowerOriginalWord, mSuggestions.get(1))) { if (!haveSufficientCommonality(mLowerOriginalWord, mSuggestions.get(1))) {
mHaveCorrection = false; mHaveCorrection = false;
} }
@ -258,7 +313,6 @@ public class Suggest implements Dictionary.WordCallback {
i++; i++;
} }
} }
removeDupes(); removeDupes();
return mSuggestions; return mSuggestions;
} }
@ -312,21 +366,50 @@ public class Suggest implements Dictionary.WordCallback {
return false; return false;
} }
public boolean addWord(final char[] word, final int offset, final int length, public boolean addWord(final char[] word, final int offset, final int length, int freq,
final int freq, final int dicTypeId) { final int dicTypeId, final Dictionary.DataType dataType) {
ArrayList<CharSequence> suggestions;
int[] priorities;
int prefMaxSuggestions;
if(dataType == Dictionary.DataType.BIGRAM) {
suggestions = mBigramSuggestions;
priorities = mBigramPriorities;
prefMaxSuggestions = mPrefMaxBigrams;
} else {
suggestions = mSuggestions;
priorities = mPriorities;
prefMaxSuggestions = mPrefMaxSuggestions;
}
int pos = 0; int pos = 0;
final int[] priorities = mPriorities;
final int prefMaxSuggestions = mPrefMaxSuggestions;
// Check if it's the same word, only caps are different // Check if it's the same word, only caps are different
if (compareCaseInsensitive(mLowerOriginalWord, word, offset, length)) { if (compareCaseInsensitive(mLowerOriginalWord, word, offset, length)) {
pos = 0; pos = 0;
} else { } else {
if (dataType == Dictionary.DataType.UNIGRAM) {
// Check if the word was already added before (by bigram data)
int bigramSuggestion = searchBigramSuggestion(word,offset,length);
if(bigramSuggestion >= 0) {
// turn freq from bigram into multiplier specified above
double multiplier = (((double) mBigramPriorities[bigramSuggestion])
/ MAXIMUM_BIGRAM_FREQUENCY)
* (BIGRAM_MULTIPLIER_MAX - BIGRAM_MULTIPLIER_MIN)
+ BIGRAM_MULTIPLIER_MIN;
/* Log.d("Suggest","bigram num: " + bigramSuggestion
+ " wordB: " + mBigramSuggestions.get(bigramSuggestion).toString()
+ " currentPriority: " + freq + " bigramPriority: "
+ mBigramPriorities[bigramSuggestion]
+ " multiplier: " + multiplier); */
freq = (int)Math.round((freq * multiplier));
}
}
// Check the last one's priority and bail // Check the last one's priority and bail
if (priorities[prefMaxSuggestions - 1] >= freq) return true; if (priorities[prefMaxSuggestions - 1] >= freq) return true;
while (pos < prefMaxSuggestions) { while (pos < prefMaxSuggestions) {
if (priorities[pos] < freq if (priorities[pos] < freq
|| (priorities[pos] == freq && length < mSuggestions || (priorities[pos] == freq && length < suggestions.get(pos).length())) {
.get(pos).length())) {
break; break;
} }
pos++; pos++;
@ -336,12 +419,13 @@ public class Suggest implements Dictionary.WordCallback {
if (pos >= prefMaxSuggestions) { if (pos >= prefMaxSuggestions) {
return true; return true;
} }
System.arraycopy(priorities, pos, priorities, pos + 1, System.arraycopy(priorities, pos, priorities, pos + 1,
prefMaxSuggestions - pos - 1); prefMaxSuggestions - pos - 1);
priorities[pos] = freq; priorities[pos] = freq;
int poolSize = mStringPool.size(); int poolSize = mStringPool.size();
StringBuilder sb = poolSize > 0 ? (StringBuilder) mStringPool.remove(poolSize - 1) StringBuilder sb = poolSize > 0 ? (StringBuilder) mStringPool.remove(poolSize - 1)
: new StringBuilder(32); : new StringBuilder(Dictionary.MAX_WORD_LENGTH);
sb.setLength(0); sb.setLength(0);
if (mCapitalize) { if (mCapitalize) {
sb.append(Character.toUpperCase(word[offset])); sb.append(Character.toUpperCase(word[offset]));
@ -351,9 +435,9 @@ public class Suggest implements Dictionary.WordCallback {
} else { } else {
sb.append(word, offset, length); sb.append(word, offset, length);
} }
mSuggestions.add(pos, sb); suggestions.add(pos, sb);
if (mSuggestions.size() > prefMaxSuggestions) { if (suggestions.size() > prefMaxSuggestions) {
CharSequence garbage = mSuggestions.remove(prefMaxSuggestions); CharSequence garbage = suggestions.remove(prefMaxSuggestions);
if (garbage instanceof StringBuilder) { if (garbage instanceof StringBuilder) {
mStringPool.add(garbage); mStringPool.add(garbage);
} }
@ -363,6 +447,26 @@ public class Suggest implements Dictionary.WordCallback {
return true; return true;
} }
private int searchBigramSuggestion(final char[] word, final int offset, final int length) {
// TODO This is almost O(n^2). Might need fix.
// search whether the word appeared in bigram data
int bigramSuggestSize = mBigramSuggestions.size();
for(int i = 0; i < bigramSuggestSize; i++) {
if(mBigramSuggestions.get(i).length() == length) {
boolean chk = true;
for(int j = 0; j < length; j++) {
if(mBigramSuggestions.get(i).charAt(j) != word[offset+j]) {
chk = false;
break;
}
}
if(chk) return i;
}
}
return -1;
}
public boolean isValidWord(final CharSequence word) { public boolean isValidWord(final CharSequence word) {
if (word == null || word.length() == 0) { if (word == null || word.length() == 0) {
return false; return false;
@ -373,21 +477,21 @@ public class Suggest implements Dictionary.WordCallback {
|| (mContactsDictionary != null && mContactsDictionary.isValidWord(word)); || (mContactsDictionary != null && mContactsDictionary.isValidWord(word));
} }
private void collectGarbage() { private void collectGarbage(ArrayList<CharSequence> suggestions, int prefMaxSuggestions) {
int poolSize = mStringPool.size(); int poolSize = mStringPool.size();
int garbageSize = mSuggestions.size(); int garbageSize = suggestions.size();
while (poolSize < mPrefMaxSuggestions && garbageSize > 0) { while (poolSize < prefMaxSuggestions && garbageSize > 0) {
CharSequence garbage = mSuggestions.get(garbageSize - 1); CharSequence garbage = suggestions.get(garbageSize - 1);
if (garbage != null && garbage instanceof StringBuilder) { if (garbage != null && garbage instanceof StringBuilder) {
mStringPool.add(garbage); mStringPool.add(garbage);
poolSize++; poolSize++;
} }
garbageSize--; garbageSize--;
} }
if (poolSize == mPrefMaxSuggestions + 1) { if (poolSize == prefMaxSuggestions + 1) {
Log.w("Suggest", "String pool got too big: " + poolSize); Log.w("Suggest", "String pool got too big: " + poolSize);
} }
mSuggestions.clear(); suggestions.clear();
} }
public void close() { public void close() {

2
java/src/com/android/inputmethod/latin/WordComposer.java
View file

@ -55,7 +55,9 @@ public class WordComposer {
mTypedWord = new StringBuilder(copy.mTypedWord); mTypedWord = new StringBuilder(copy.mTypedWord);
mCapsCount = copy.mCapsCount; mCapsCount = copy.mCapsCount;
mAutoCapitalized = copy.mAutoCapitalized; mAutoCapitalized = copy.mAutoCapitalized;
mIsCapitalized = copy.mIsCapitalized;
} }
/** /**
* Clear out the keys registered so far. * Clear out the keys registered so far.
*/ */

4
native/Android.mk
View file

@ -8,8 +8,8 @@ LOCAL_SRC_FILES := \
src/dictionary.cpp \ src/dictionary.cpp \
src/char_utils.cpp src/char_utils.cpp
LOCAL_C_INCLUDES += \ LOCAL_NDK_VERSION := 4
$(JNI_H_INCLUDE) LOCAL_SDK_VERSION := 8
LOCAL_PRELINK_MODULE := false LOCAL_PRELINK_MODULE := false

28
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -59,8 +59,7 @@ static int latinime_BinaryDictionary_getSuggestions(
jint maxAlternatives, jint skipPos, jintArray nextLettersArray, jint nextLettersSize) jint maxAlternatives, jint skipPos, jintArray nextLettersArray, jint nextLettersSize)
{ {
Dictionary *dictionary = (Dictionary*) dict; Dictionary *dictionary = (Dictionary*) dict;
if (dictionary == NULL) if (dictionary == NULL) return 0;
return 0;
int *frequencies = env->GetIntArrayElements(frequencyArray, NULL); int *frequencies = env->GetIntArrayElements(frequencyArray, NULL);
int *inputCodes = env->GetIntArrayElements(inputArray, NULL); int *inputCodes = env->GetIntArrayElements(inputArray, NULL);
@ -81,6 +80,28 @@ static int latinime_BinaryDictionary_getSuggestions(
return count; return count;
} }
static int latinime_BinaryDictionary_getBigrams
(JNIEnv *env, jobject object, jint dict, jcharArray wordArray, jint wordLength,
jcharArray outputArray, jintArray frequencyArray, jint maxWordLength, jint maxBigrams)
{
Dictionary *dictionary = (Dictionary*) dict;
if (dictionary == NULL) return 0;
jchar *word = env->GetCharArrayElements(wordArray, NULL);
jchar *outputChars = env->GetCharArrayElements(outputArray, NULL);
int *frequencies = env->GetIntArrayElements(frequencyArray, NULL);
int count = dictionary->getBigrams((unsigned short*) word, wordLength,
(unsigned short*) outputChars, frequencies, maxWordLength, maxBigrams);
env->ReleaseCharArrayElements(wordArray, word, JNI_ABORT);
env->ReleaseCharArrayElements(outputArray, outputChars, 0);
env->ReleaseIntArrayElements(frequencyArray, frequencies, 0);
return count;
}
static jboolean latinime_BinaryDictionary_isValidWord static jboolean latinime_BinaryDictionary_isValidWord
(JNIEnv *env, jobject object, jint dict, jcharArray wordArray, jint wordLength) (JNIEnv *env, jobject object, jint dict, jcharArray wordArray, jint wordLength)
{ {
@ -108,7 +129,8 @@ static JNINativeMethod gMethods[] = {
(void*)latinime_BinaryDictionary_open}, (void*)latinime_BinaryDictionary_open},
{"closeNative", "(I)V", (void*)latinime_BinaryDictionary_close}, {"closeNative", "(I)V", (void*)latinime_BinaryDictionary_close},
{"getSuggestionsNative", "(I[II[C[IIIII[II)I", (void*)latinime_BinaryDictionary_getSuggestions}, {"getSuggestionsNative", "(I[II[C[IIIII[II)I", (void*)latinime_BinaryDictionary_getSuggestions},
{"isValidWordNative", "(I[CI)Z", (void*)latinime_BinaryDictionary_isValidWord} {"isValidWordNative", "(I[CI)Z", (void*)latinime_BinaryDictionary_isValidWord},
{"getBigramsNative", "(I[CI[C[III)I", (void*)latinime_BinaryDictionary_getBigrams}
}; };
static int registerNativeMethods(JNIEnv* env, const char* className, static int registerNativeMethods(JNIEnv* env, const char* className,

281
native/src/dictionary.cpp
View file

@ -19,6 +19,7 @@
#include <fcntl.h> #include <fcntl.h>
#include <sys/mman.h> #include <sys/mman.h>
#include <string.h> #include <string.h>
//#define LOG_TAG "dictionary.cpp"
//#include <cutils/log.h> //#include <cutils/log.h>
#define LOGI #define LOGI
@ -27,6 +28,9 @@
#include "char_utils.h" #include "char_utils.h"
#define DEBUG_DICT 0 #define DEBUG_DICT 0
#define DICTIONARY_VERSION_MIN 200
#define DICTIONARY_HEADER_SIZE 2
#define NOT_VALID_WORD -99
namespace latinime { namespace latinime {
@ -35,6 +39,7 @@ Dictionary::Dictionary(void *dict, int typedLetterMultiplier, int fullWordMultip
mDict = (unsigned char*) dict; mDict = (unsigned char*) dict;
mTypedLetterMultiplier = typedLetterMultiplier; mTypedLetterMultiplier = typedLetterMultiplier;
mFullWordMultiplier = fullWordMultiplier; mFullWordMultiplier = fullWordMultiplier;
getVersionNumber();
} }
Dictionary::~Dictionary() Dictionary::~Dictionary()
@ -58,7 +63,11 @@ int Dictionary::getSuggestions(int *codes, int codesSize, unsigned short *outWor
mNextLettersFrequencies = nextLetters; mNextLettersFrequencies = nextLetters;
mNextLettersSize = nextLettersSize; mNextLettersSize = nextLettersSize;
getWordsRec(0, 0, mInputLength * 3, false, 1, 0, 0); if (checkIfDictVersionIsLatest()) {
getWordsRec(DICTIONARY_HEADER_SIZE, 0, mInputLength * 3, false, 1, 0, 0);
} else {
getWordsRec(0, 0, mInputLength * 3, false, 1, 0, 0);
}
// Get the word count // Get the word count
suggWords = 0; suggWords = 0;
@ -85,6 +94,21 @@ Dictionary::registerNextLetter(unsigned short c)
} }
} }
void
Dictionary::getVersionNumber()
{
mVersion = (mDict[0] & 0xFF);
mBigram = (mDict[1] & 0xFF);
LOGI("IN NATIVE SUGGEST Version: %d Bigram : %d \n", mVersion, mBigram);
}
// Checks whether it has the latest dictionary or the old dictionary
bool
Dictionary::checkIfDictVersionIsLatest()
{
return (mVersion >= DICTIONARY_VERSION_MIN) && (mBigram == 1 || mBigram == 0);
}
unsigned short unsigned short
Dictionary::getChar(int *pos) Dictionary::getChar(int *pos)
{ {
@ -112,6 +136,28 @@ Dictionary::getAddress(int *pos)
return address; return address;
} }
int
Dictionary::getFreq(int *pos)
{
int freq = mDict[(*pos)++] & 0xFF;
if (checkIfDictVersionIsLatest()) {
// skipping bigram
int bigramExist = (mDict[*pos] & FLAG_BIGRAM_READ);
if (bigramExist > 0) {
int nextBigramExist = 1;
while (nextBigramExist > 0) {
(*pos) += 3;
nextBigramExist = (mDict[(*pos)++] & FLAG_BIGRAM_CONTINUED);
}
} else {
(*pos)++;
}
}
return freq;
}
int int
Dictionary::wideStrLen(unsigned short *str) Dictionary::wideStrLen(unsigned short *str)
{ {
@ -161,6 +207,46 @@ Dictionary::addWord(unsigned short *word, int length, int frequency)
return false; return false;
} }
bool
Dictionary::addWordBigram(unsigned short *word, int length, int frequency)
{
word[length] = 0;
if (DEBUG_DICT) {
char s[length + 1];
for (int i = 0; i <= length; i++) s[i] = word[i];
LOGI("Bigram: Found word = %s, freq = %d : \n", s, frequency);
}
// Find the right insertion point
int insertAt = 0;
while (insertAt < mMaxBigrams) {
if (frequency > mBigramFreq[insertAt]
|| (mBigramFreq[insertAt] == frequency
&& length < wideStrLen(mBigramChars + insertAt * mMaxWordLength))) {
break;
}
insertAt++;
}
LOGI("Bigram: InsertAt -> %d maxBigrams: %d\n", insertAt, mMaxBigrams);
if (insertAt < mMaxBigrams) {
memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]),
(char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]),
(mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0]));
mBigramFreq[insertAt] = frequency;
memmove((char*) mBigramChars + (insertAt + 1) * mMaxWordLength * sizeof(short),
(char*) mBigramChars + (insertAt ) * mMaxWordLength * sizeof(short),
(mMaxBigrams - insertAt - 1) * sizeof(short) * mMaxWordLength);
unsigned short *dest = mBigramChars + (insertAt ) * mMaxWordLength;
while (length--) {
*dest++ = *word++;
}
*dest = 0; // NULL terminate
if (DEBUG_DICT) LOGI("Bigram: Added word at %d\n", insertAt);
return true;
}
return false;
}
unsigned short unsigned short
Dictionary::toLowerCase(unsigned short c) { Dictionary::toLowerCase(unsigned short c) {
if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) { if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
@ -213,12 +299,17 @@ Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int s
} }
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
// -- at char
unsigned short c = getChar(&pos); unsigned short c = getChar(&pos);
// -- at flag/add
unsigned short lowerC = toLowerCase(c); unsigned short lowerC = toLowerCase(c);
bool terminal = getTerminal(&pos); bool terminal = getTerminal(&pos);
int childrenAddress = getAddress(&pos); int childrenAddress = getAddress(&pos);
// -- after address or flag
int freq = 1; int freq = 1;
if (terminal) freq = getFreq(&pos); if (terminal) freq = getFreq(&pos);
// -- after add or freq
// If we are only doing completions, no need to look at the typed characters. // If we are only doing completions, no need to look at the typed characters.
if (completion) { if (completion) {
mWord[depth] = c; mWord[depth] = c;
@ -232,7 +323,7 @@ Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int s
getWordsRec(childrenAddress, depth + 1, maxDepth, getWordsRec(childrenAddress, depth + 1, maxDepth,
completion, snr, inputIndex, diffs); completion, snr, inputIndex, diffs);
} }
} else if (c == QUOTE && currentChars[0] != QUOTE || mSkipPos == depth) { } else if ((c == QUOTE && currentChars[0] != QUOTE) || mSkipPos == depth) {
// Skip the ' or other letter and continue deeper // Skip the ' or other letter and continue deeper
mWord[depth] = c; mWord[depth] = c;
if (childrenAddress != 0) { if (childrenAddress != 0) {
@ -270,14 +361,185 @@ Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int s
} }
} }
bool int
Dictionary::isValidWord(unsigned short *word, int length) Dictionary::getBigramAddress(int *pos, bool advance)
{ {
return isValidWordRec(0, word, 0, length); int address = 0;
address += (mDict[*pos] & 0x3F) << 16;
address += (mDict[*pos + 1] & 0xFF) << 8;
address += (mDict[*pos + 2] & 0xFF);
if (advance) {
*pos += 3;
}
return address;
}
int
Dictionary::getBigramFreq(int *pos)
{
int freq = mDict[(*pos)++] & FLAG_BIGRAM_FREQ;
return freq;
}
int
Dictionary::getBigrams(unsigned short *prevWord, int prevWordLength, unsigned short *bigramChars,
int *bigramFreq, int maxWordLength, int maxBigrams)
{
mBigramFreq = bigramFreq;
mBigramChars = bigramChars;
mMaxWordLength = maxWordLength;
mMaxBigrams = maxBigrams;
if (mBigram == 1 && checkIfDictVersionIsLatest()) {
int pos = isValidWordRec(DICTIONARY_HEADER_SIZE, prevWord, 0, prevWordLength);
LOGI("Pos -> %d\n", pos);
if (pos < 0) {
return 0;
}
int bigramCount = 0;
int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
if (bigramExist > 0) {
int nextBigramExist = 1;
while (nextBigramExist > 0) {
int bigramAddress = getBigramAddress(&pos, true);
int frequency = (FLAG_BIGRAM_FREQ & mDict[pos]);
// search for all bigrams and store them
searchForTerminalNode(bigramAddress, frequency);
nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
bigramCount++;
}
}
return bigramCount;
}
return 0;
}
void
Dictionary::searchForTerminalNode(int addressLookingFor, int frequency)
{
// track word with such address and store it in an array
unsigned short word[mMaxWordLength];
int pos;
int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
bool found = false;
char followingChar = ' ';
int depth = -1;
while(!found) {
bool followDownAddressSearchStop = false;
bool firstAddress = true;
bool haveToSearchAll = true;
if (depth >= 0) {
word[depth] = (unsigned short) followingChar;
}
pos = followDownBranchAddress; // pos start at count
int count = mDict[pos] & 0xFF;
LOGI("count - %d\n",count);
pos++;
for (int i = 0; i < count; i++) {
// pos at data
pos++;
// pos now at flag
if (!getFirstBitOfByte(&pos)) { // non-terminal
if (!followDownAddressSearchStop) {
int addr = getBigramAddress(&pos, false);
if (addr > addressLookingFor) {
followDownAddressSearchStop = true;
if (firstAddress) {
firstAddress = false;
haveToSearchAll = true;
} else if (!haveToSearchAll) {
break;
}
} else {
followDownBranchAddress = addr;
followingChar = (char)(0xFF & mDict[pos-1]);
if (firstAddress) {
firstAddress = false;
haveToSearchAll = false;
}
}
}
pos += 3;
} else if (getFirstBitOfByte(&pos)) { // terminal
if (addressLookingFor == (pos-1)) { // found !!
depth++;
word[depth] = (0xFF & mDict[pos-1]);
found = true;
break;
}
if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
if (!followDownAddressSearchStop) {
int addr = getBigramAddress(&pos, false);
if (addr > addressLookingFor) {
followDownAddressSearchStop = true;
if (firstAddress) {
firstAddress = false;
haveToSearchAll = true;
} else if (!haveToSearchAll) {
break;
}
} else {
followDownBranchAddress = addr;
followingChar = (char)(0xFF & mDict[pos-1]);
if (firstAddress) {
firstAddress = false;
haveToSearchAll = true;
}
}
}
pos += 4;
} else { // freq only (2 byte)
pos += 2;
}
// skipping bigram
int bigramExist = (mDict[pos] & FLAG_BIGRAM_READ);
if (bigramExist > 0) {
int nextBigramExist = 1;
while (nextBigramExist > 0) {
pos += 3;
nextBigramExist = (mDict[pos++] & FLAG_BIGRAM_CONTINUED);
}
} else {
pos++;
}
}
}
depth++;
if (followDownBranchAddress == 0) {
LOGI("ERROR!!! Cannot find bigram!!");
break;
}
}
addWordBigram(word, depth, frequency);
} }
bool bool
Dictionary::isValidWord(unsigned short *word, int length)
{
if (checkIfDictVersionIsLatest()) {
return (isValidWordRec(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD);
} else {
return (isValidWordRec(0, word, 0, length) != NOT_VALID_WORD);
}
}
int
Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) { Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length) {
// returns address of bigram data of that word
// return -99 if not found
int count = getCount(&pos); int count = getCount(&pos);
unsigned short currentChar = (unsigned short) word[offset]; unsigned short currentChar = (unsigned short) word[offset];
for (int j = 0; j < count; j++) { for (int j = 0; j < count; j++) {
@ -287,12 +549,13 @@ Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length
if (c == currentChar) { if (c == currentChar) {
if (offset == length - 1) { if (offset == length - 1) {
if (terminal) { if (terminal) {
return true; return (pos+1);
} }
} else { } else {
if (childPos != 0) { if (childPos != 0) {
if (isValidWordRec(childPos, word, offset + 1, length)) { int t = isValidWordRec(childPos, word, offset + 1, length);
return true; if (t > 0) {
return t;
} }
} }
} }
@ -303,7 +566,7 @@ Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length
// There could be two instances of each alphabet - upper and lower case. So continue // There could be two instances of each alphabet - upper and lower case. So continue
// looking ... // looking ...
} }
return false; return NOT_VALID_WORD;
} }

25
native/src/dictionary.h
View file

@ -28,12 +28,19 @@ namespace latinime {
// if the word has other endings. // if the word has other endings.
#define FLAG_TERMINAL_MASK 0x80 #define FLAG_TERMINAL_MASK 0x80
#define FLAG_BIGRAM_READ 0x80
#define FLAG_BIGRAM_CHILDEXIST 0x40
#define FLAG_BIGRAM_CONTINUED 0x80
#define FLAG_BIGRAM_FREQ 0x7F
class Dictionary { class Dictionary {
public: public:
Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier); Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier);
int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies, int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
int maxWordLength, int maxWords, int maxAlternatives, int skipPos, int maxWordLength, int maxWords, int maxAlternatives, int skipPos,
int *nextLetters, int nextLettersSize); int *nextLetters, int nextLettersSize);
int getBigrams(unsigned short *word, int length, unsigned short *outWords, int *frequencies,
int maxWordLength, int maxBigrams);
bool isValidWord(unsigned short *word, int length); bool isValidWord(unsigned short *word, int length);
void setAsset(void *asset) { mAsset = asset; } void setAsset(void *asset) { mAsset = asset; }
void *getAsset() { return mAsset; } void *getAsset() { return mAsset; }
@ -41,28 +48,40 @@ public:
private: private:
void getVersionNumber();
bool checkIfDictVersionIsLatest();
int getAddress(int *pos); int getAddress(int *pos);
int getBigramAddress(int *pos, bool advance);
int getFreq(int *pos);
int getBigramFreq(int *pos);
void searchForTerminalNode(int address, int frequency);
bool getFirstBitOfByte(int *pos) { return (mDict[*pos] & 0x80) > 0; }
bool getSecondBitOfByte(int *pos) { return (mDict[*pos] & 0x40) > 0; }
bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; } bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; }
int getFreq(int *pos) { return mDict[(*pos)++] & 0xFF; }
int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; } int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; }
unsigned short getChar(int *pos); unsigned short getChar(int *pos);
int wideStrLen(unsigned short *str); int wideStrLen(unsigned short *str);
bool sameAsTyped(unsigned short *word, int length); bool sameAsTyped(unsigned short *word, int length);
bool addWord(unsigned short *word, int length, int frequency); bool addWord(unsigned short *word, int length, int frequency);
bool addWordBigram(unsigned short *word, int length, int frequency);
unsigned short toLowerCase(unsigned short c); unsigned short toLowerCase(unsigned short c);
void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency, void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency,
int inputIndex, int diffs); int inputIndex, int diffs);
bool isValidWordRec(int pos, unsigned short *word, int offset, int length); int isValidWordRec(int pos, unsigned short *word, int offset, int length);
void registerNextLetter(unsigned short c); void registerNextLetter(unsigned short c);
unsigned char *mDict; unsigned char *mDict;
void *mAsset; void *mAsset;
int *mFrequencies; int *mFrequencies;
int *mBigramFreq;
int mMaxWords; int mMaxWords;
int mMaxBigrams;
int mMaxWordLength; int mMaxWordLength;
unsigned short *mOutputChars; unsigned short *mOutputChars;
unsigned short *mBigramChars;
int *mInputCodes; int *mInputCodes;
int mInputLength; int mInputLength;
int mMaxAlternatives; int mMaxAlternatives;
@ -74,6 +93,8 @@ private:
int mTypedLetterMultiplier; int mTypedLetterMultiplier;
int *mNextLettersFrequencies; int *mNextLettersFrequencies;
int mNextLettersSize; int mNextLettersSize;
int mVersion;
int mBigram;
}; };
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------

36
tests/data/bigramlist.xml Normal file
View file

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
/*
**
** Copyright 2010, The Android Open Source Project
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
*/
-->
<bigrams>
<bi w1="I'm" count="1">
<w w2="about" p="100" />
</bi>
<bi w1="about" count="3">
<w w2="part" p="117" />
<w w2="business" p="100" />
<w w2="being" p="10" />
</bi>
<bi w1="business" count="1">
<w w2="people" p="100" />
</bi>
<bi w1="from" count="1">
<w w2="same" p="117" />
</bi>
</bigrams>

1
tests/data/wordlist.xml
View file

@ -225,6 +225,7 @@
<w f="179">services</w> <w f="179">services</w>
<w f="170">niño</w> <w f="170">niño</w>
<w f="170">María</w> <w f="170">María</w>
<w f="70">car</w>
<w f="0">hmmm</w> <w f="0">hmmm</w>
<w f="0">hon</w> <w f="0">hon</w>
<w f="0">tty</w> <w f="0">tty</w>

BIN
tests/res/raw/test.dict
View file

Binary file not shown.

67
tests/src/com/android/inputmethod/latin/tests/SuggestTests.java
View file

@ -71,7 +71,7 @@ public class SuggestTests extends AndroidTestCase {
Log.w(TAG, "No available size for binary dictionary"); Log.w(TAG, "No available size for binary dictionary");
} }
mSuggest.setAutoTextEnabled(false); mSuggest.setAutoTextEnabled(false);
mSuggest.setCorrectionMode(Suggest.CORRECTION_FULL); mSuggest.setCorrectionMode(Suggest.CORRECTION_FULL_BIGRAM);
} }
/************************** Helper functions ************************/ /************************** Helper functions ************************/
@ -108,19 +108,56 @@ public class SuggestTests extends AndroidTestCase {
private boolean isDefaultSuggestion(CharSequence typed, CharSequence expected) { private boolean isDefaultSuggestion(CharSequence typed, CharSequence expected) {
WordComposer word = createWordComposer(typed); WordComposer word = createWordComposer(typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false); List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, null);
return isDefaultSuggestion(suggestions, expected);
}
private void getBigramSuggestions(CharSequence previous, CharSequence typed) {
if(!TextUtils.isEmpty(previous) && (typed.length() > 1)) {
WordComposer firstChar = createWordComposer(typed.charAt(0) + "");
mSuggest.getSuggestions(null, firstChar, false, previous);
}
}
private boolean isDefaultNextSuggestion(CharSequence previous, CharSequence typed,
CharSequence expected) {
WordComposer word = createWordComposer(typed);
getBigramSuggestions(previous, typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, previous);
return isDefaultSuggestion(suggestions, expected); return isDefaultSuggestion(suggestions, expected);
} }
private boolean isDefaultCorrection(CharSequence typed, CharSequence expected) { private boolean isDefaultCorrection(CharSequence typed, CharSequence expected) {
WordComposer word = createWordComposer(typed); WordComposer word = createWordComposer(typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false); List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, null);
return isDefaultSuggestion(suggestions, expected) && mSuggest.hasMinimalCorrection();
}
private boolean isDefaultNextCorrection(CharSequence previous, CharSequence typed,
CharSequence expected) {
WordComposer word = createWordComposer(typed);
getBigramSuggestions(previous, typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, previous);
for(int i=0;i<suggestions.size();i++) {
Log.i(TAG,i+" "+suggestions.get(i));
}
return isDefaultSuggestion(suggestions, expected) && mSuggest.hasMinimalCorrection(); return isDefaultSuggestion(suggestions, expected) && mSuggest.hasMinimalCorrection();
} }
private boolean isASuggestion(CharSequence typed, CharSequence expected) { private boolean isASuggestion(CharSequence typed, CharSequence expected) {
WordComposer word = createWordComposer(typed); WordComposer word = createWordComposer(typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false); List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, null);
for (int i = 1; i < suggestions.size(); i++) {
if (TextUtils.equals(suggestions.get(i), expected)) return true;
}
return false;
}
private boolean isASuggestion(CharSequence previous, CharSequence typed,
CharSequence expected) {
WordComposer word = createWordComposer(typed);
getBigramSuggestions(previous, typed);
List<CharSequence> suggestions = mSuggest.getSuggestions(null, word, false, previous);
for (int i = 1; i < suggestions.size(); i++) { for (int i = 1; i < suggestions.size(); i++) {
if (TextUtils.equals(suggestions.get(i), expected)) return true; if (TextUtils.equals(suggestions.get(i), expected)) return true;
} }
@ -248,4 +285,26 @@ public class SuggestTests extends AndroidTestCase {
// Mar<LATIN SMALL LETTER I WITH ACUTE>a // Mar<LATIN SMALL LETTER I WITH ACUTE>a
assertTrue(isDefaultCorrection("maria", "Mar\u00EDa")); assertTrue(isDefaultCorrection("maria", "Mar\u00EDa"));
} }
/**
* Make sure bigrams are showing when first character is typed
* and don't show any when there aren't any
*/
public void testBigramsAtFirstChar() {
assertTrue(isDefaultNextCorrection("about", "p", "part"));
assertTrue(isDefaultNextCorrection("I'm", "a", "about"));
assertTrue(isDefaultNextCorrection("about", "b", "business"));
assertTrue(isASuggestion("about", "b", "being"));
assertFalse(isDefaultNextSuggestion("about", "p", "business"));
}
/**
* Make sure bigrams score affects the original score
*/
public void testBigramsScoreEffect() {
assertTrue(isDefaultCorrection("pa", "page"));
assertTrue(isDefaultNextCorrection("about", "pa", "part"));
assertTrue(isDefaultCorrection("sa", "said"));
assertTrue(isDefaultNextCorrection("from", "sa", "same"));
}
} }