From e4e1130d003a75e3b5cbea1678755d2b71d7cb1d Mon Sep 17 00:00:00 2001 From: Amith Yamasani Date: Wed, 28 Apr 2010 18:12:58 -0700 Subject: [PATCH] Tests and some new constructors to help in testing. Added tests for the dictionary lookup and correction logic on the primary dictionary. This exercises part of the Suggest class and the native dictionary lookup code. --- java/proguard.flags | 5 + .../inputmethod/latin/BinaryDictionary.java | 20 ++ .../android/inputmethod/latin/Suggest.java | 21 +- .../inputmethod/latin/WordComposer.java | 2 +- tests/Android.mk | 17 ++ tests/AndroidManifest.xml | 33 +++ tests/data/wordlist.xml | 243 +++++++++++++++++ tests/res/raw/test.dict | Bin 0 -> 2562 bytes .../inputmethod/latin/tests/SuggestTests.java | 248 ++++++++++++++++++ 9 files changed, 581 insertions(+), 8 deletions(-) create mode 100644 tests/Android.mk create mode 100644 tests/AndroidManifest.xml create mode 100644 tests/data/wordlist.xml create mode 100644 tests/res/raw/test.dict create mode 100644 tests/src/com/android/inputmethod/latin/tests/SuggestTests.java diff --git a/java/proguard.flags b/java/proguard.flags index 0a5d2dda9..829a096c0 100644 --- a/java/proguard.flags +++ b/java/proguard.flags @@ -1,3 +1,8 @@ -keep class com.android.inputmethod.latin.BinaryDictionary { int mDictLength; + (...); +} + +-keep class com.android.inputmethod.latin.Suggest { + (...); } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 4901b210b..6473f4558 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -68,6 +68,26 @@ public class BinaryDictionary extends Dictionary { } } + /** + * Create a dictionary from a byte buffer. This is used for testing. + * @param context application context for reading resources + * @param resId the resource containing the raw binary dictionary + */ + public BinaryDictionary(Context context, ByteBuffer byteBuffer) { + if (byteBuffer != null) { + if (byteBuffer.isDirect()) { + mNativeDictDirectBuffer = byteBuffer; + } else { + mNativeDictDirectBuffer = ByteBuffer.allocateDirect(byteBuffer.capacity()); + byteBuffer.rewind(); + mNativeDictDirectBuffer.put(byteBuffer); + } + mDictLength = byteBuffer.capacity(); + mNativeDict = openNative(mNativeDictDirectBuffer, + TYPED_LETTER_MULTIPLIER, FULL_WORD_FREQ_MULTIPLIER); + } + } + private native int openNative(ByteBuffer bb, int typedLetterMultiplier, int fullWordMultiplier); private native void closeNative(int dict); private native boolean isValidWordNative(int nativeData, char[] word, int wordLength); diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index a70bea003..010913d6d 100755 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -16,18 +16,17 @@ package com.android.inputmethod.latin; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import android.content.Context; import android.text.AutoText; import android.text.TextUtils; import android.util.Log; import android.view.View; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.android.inputmethod.latin.WordComposer; - /** * This class loads a dictionary and provides a list of suggestions for a given sequence of * characters. This includes corrections and completions. @@ -69,9 +68,17 @@ public class Suggest implements Dictionary.WordCallback { private int mCorrectionMode = CORRECTION_BASIC; - public Suggest(Context context, int dictionaryResId) { mMainDict = new BinaryDictionary(context, dictionaryResId); + initPool(); + } + + public Suggest(Context context, ByteBuffer byteBuffer) { + mMainDict = new BinaryDictionary(context, byteBuffer); + initPool(); + } + + private void initPool() { for (int i = 0; i < mPrefMaxSuggestions; i++) { StringBuilder sb = new StringBuilder(32); mStringPool.add(sb); diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 19f714ae7..2547aa133 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -44,7 +44,7 @@ public class WordComposer { */ private boolean mIsCapitalized; - WordComposer() { + public WordComposer() { mCodes = new ArrayList(12); mTypedWord = new StringBuilder(20); } diff --git a/tests/Android.mk b/tests/Android.mk new file mode 100644 index 000000000..fba7a8d74 --- /dev/null +++ b/tests/Android.mk @@ -0,0 +1,17 @@ +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +# We only want this apk build for tests. +LOCAL_MODULE_TAGS := tests +LOCAL_CERTIFICATE := shared + +LOCAL_JAVA_LIBRARIES := android.test.runner + +# Include all test java files. +LOCAL_SRC_FILES := $(call all-java-files-under, src) + +LOCAL_PACKAGE_NAME := LatinIMETests + +LOCAL_INSTRUMENTATION_FOR := LatinIME + +include $(BUILD_PACKAGE) diff --git a/tests/AndroidManifest.xml b/tests/AndroidManifest.xml new file mode 100644 index 000000000..210e81489 --- /dev/null +++ b/tests/AndroidManifest.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + diff --git a/tests/data/wordlist.xml b/tests/data/wordlist.xml new file mode 100644 index 000000000..22d0caa38 --- /dev/null +++ b/tests/data/wordlist.xml @@ -0,0 +1,243 @@ + + the + and + of + to + in + that + for + with + on + it + this + you + is + was + by + or + from + but + be + Sunday + are + he + so + not + have + as + all + his + my + if + which + they + at + it's + an + your + will + about + I'm + there + had + has + when + no + were + what + more + out + just + their + up + would + here + can + who + her + me + now + our + do + some + been + two + like + them + new + time + we + she + one + over + may + any + him + calling + other + how + see + because + then + right + into + well + very + said + people + these + than + only + back + first + dot + after + where + please + could + its + before + us + again + home + also + that's + think + three + good + get + know + thank + should + going + down + last + today + those + go + through + such + don't + did + most + day + man + number + work + too + show + made + even + being + make + give + off + com + much + great + take + call + way + four + say + information + under + page + many + little + thanks + okay + five + we're + between + use + come + years + office + house + search + free + next + without + still + around + I've + business + part + every + bye + upon + you're + state + life + year + thing + since + things + something + long + got + while + I'll + help + service + really + must + does + name + both + six + want + same + each + yet + let + view + place + another + company + talk + might + am + though + find + details + look + world + old + called + case + system + news + used + contact + never + seven + city + until + during + set + why + point + twenty + high + love + services + niño + María + hmmm + hon + tty + ttyl + txt + ur + wah + whatcha + woah + ya + yea + yeh + yessir + yikes + yrs + diff --git a/tests/res/raw/test.dict b/tests/res/raw/test.dict new file mode 100644 index 0000000000000000000000000000000000000000..e789aaa9a66cd003e1282870a79fc12a8d731fdd GIT binary patch literal 2562 zcmX9=eT-aH6@Ta6`@WK5QWI*@CRRl)h$v`5lp0Slp;|-`geW37otfR4&dhtwo7tUR z4dxmRNR1G+Adr-r7O6BMmD!ou+0WZ#{X)^4*~x@g<5DNB$FZD>P5sK2}ZF}wGk zd(Q9t&c{9XEw8YXPcbDi)5ScCVh4-iDoeMQO9kWc)g>-l<8rL+^li7UQDqwnrG)<2|H)O=Mt}1Ph(w!tXa|%ksd3vGe}we5+!;P z|M!ZDaVB4jjB{i!tzzd8mGvkUjB{)RtClG16A?S7L_v>s8s|R^2pb~H&igSt7hyk2 zT}+BPnL^6xVNqsU2FY?94O{6-Nb4fQbY-ju352hvUZ}--BFnUfBBr&35N?9BMkS_O zVYpuunC>Yu{REoygdMmI%K9|8Tt6eB7e&)MS>aCp!{ zyl$;e!+MX6;yfY$?^C2r&|$dIL^L~u=j|Ho=S8Z_~I7ekAthY%kF-J-9C5|8K|{Du}{Iz{R^H{%# zwh(ES8-Ws`m8RjaAtHSsVfj<6_fW;iUriBI9zx#a!<5nvsLE|H*s+S`PAE;m)fn<3 z|B6jZg1|5~`{7}lptctQOrpVuus&$B3_p$wu5V;nS-d=}D-u>;p)MoeK8`FM0`R*Q z@B;ugRm!NPPDR&YR%W$=5~DuWM0Wd;CiMvY6R5h{ zLcn~&>OKW5+ZSpl=qek(?RQnDPTa;Ss}YMjX8*KpYY-3*JPKdP3J!Rqun{&0Xx?8X+f^9>(PNvFJw+yFW(vIYDgPU(V*(eJ0E9Xo@}2 z#on?Cdn@zoeXWE{RHVL3-*Q;y=C2;vtzJ^thpZ|Yc&ZTA3+;q zuzwVgNGI}^W^V<|hXoEV2kC3z(Yp$r*aSuII_NY7UV|HZ{;#_Z0-&;Yn?1f0*xua; zzR!(~SDC`@X9)U9;5-Ms2D)Rms@t+^yoV-w*!yiad%pvxKC%}J+K*API8VK8w1K^+ zt>{^ZhL9=mg+*6q*c+r+@3JpM%-(EPw}HNZ2TA4Gzf>gbodBJAC{F=(NMW2t#(Q@t z!`^=j>^qAyDT_?vhb!t|Qf7ZCA|8Nt1Eb$xjxzWwv8&@@@h_JpeXyJT)kXHNhmF1e z4Rn{k7E${*L2XKHGX6c2W#eZ}#C~kAOw4`?DMn1jFZ3h1HM}K$Plf$oASVBQ#MDA< z{0D$mlPUK92#c{i2U13Q60UXte3XjDe|7+s@Lzz@AezE&qABdu?*o-Sj0FF1$@p`a zME=_-ls}6jgP!T%#Qv)U+y!g&CStCmKK?l?dXEtGMG&t+6a+Y{p<2PG+Hn8P6bI|F z9H6sT63GQ30Dn1Kx zR#XCuAOZCniWEEu&0R4EHGm$F1rD~r{_zeDo4n!QkLl9q}(;R+? I_hHxn0FARwX8-^I literal 0 HcmV?d00001 diff --git a/tests/src/com/android/inputmethod/latin/tests/SuggestTests.java b/tests/src/com/android/inputmethod/latin/tests/SuggestTests.java new file mode 100644 index 000000000..9401d926a --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/tests/SuggestTests.java @@ -0,0 +1,248 @@ +package com.android.inputmethod.latin.tests; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.util.List; + +import android.content.Context; +import android.test.AndroidTestCase; +import android.text.TextUtils; +import android.util.Log; + +import com.android.inputmethod.latin.Suggest; +import com.android.inputmethod.latin.WordComposer; + +public class SuggestTests extends AndroidTestCase { + private static final String TAG = "SuggestTests"; + + private Suggest mSuggest; + + int[][] adjacents = { + {'a','s','w','q',-1}, + {'b','h','v','n','g','j',-1}, + {'c','v','f','x','g',}, + {'d','f','r','e','s','x',-1}, + {'e','w','r','s','d',-1}, + {'f','g','d','c','t','r',-1}, + {'g','h','f','y','t','v',-1}, + {'h','j','u','g','b','y',-1}, + {'i','o','u','k',-1}, + {'j','k','i','h','u','n',-1}, + {'k','l','o','j','i','m',-1}, + {'l','k','o','p',-1}, + {'m','k','n','l',-1}, + {'n','m','j','k','b',-1}, + {'o','p','i','l',-1}, + {'p','o',-1}, + {'q','w',-1}, + {'r','t','e','f',-1}, + {'s','d','e','w','a','z',-1}, + {'t','y','r',-1}, + {'u','y','i','h','j',-1}, + {'v','b','g','c','h',-1}, + {'w','e','q',-1}, + {'x','c','d','z','f',-1}, + {'y','u','t','h','g',-1}, + {'z','s','x','a','d',-1}, + }; + + @Override + protected void setUp() { + final Context context = getTestContext(); + InputStream is = context.getResources().openRawResource(R.raw.test); + Log.i(TAG, "Stream type is " + is); + try { + int avail = is.available(); + if (avail > 0) { + ByteBuffer byteBuffer = + ByteBuffer.allocateDirect(avail).order(ByteOrder.nativeOrder()); + int got = Channels.newChannel(is).read(byteBuffer); + if (got != avail) { + Log.e(TAG, "Read " + got + " bytes, expected " + avail); + } else { + mSuggest = new Suggest(context, byteBuffer); + Log.i(TAG, "Created mSuggest " + avail + " bytes"); + } + } + } catch (IOException ioe) { + Log.w(TAG, "No available size for binary dictionary"); + } + mSuggest.setAutoTextEnabled(false); + mSuggest.setCorrectionMode(Suggest.CORRECTION_FULL); + } + + /************************** Helper functions ************************/ + + private WordComposer createWordComposer(CharSequence s) { + WordComposer word = new WordComposer(); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + int[] codes; + // If it's not a lowercase letter, don't find adjacent letters + if (c < 'a' || c > 'z') { + codes = new int[] { c }; + } else { + codes = adjacents[c - 'a']; + } + word.add(c, codes); + } + return word; + } + + private void showList(String title, List suggestions) { + Log.i(TAG, title); + for (int i = 0; i < suggestions.size(); i++) { + Log.i(title, suggestions.get(i) + ", "); + } + } + + private boolean isDefaultSuggestion(List suggestions, CharSequence word) { + // Check if either the word is what you typed or the first alternative + return suggestions.size() > 0 && + (/*TextUtils.equals(suggestions.get(0), word) || */ + (suggestions.size() > 1 && TextUtils.equals(suggestions.get(1), word))); + } + + private boolean isDefaultSuggestion(CharSequence typed, CharSequence expected) { + WordComposer word = createWordComposer(typed); + List suggestions = mSuggest.getSuggestions(null, word, false); + return isDefaultSuggestion(suggestions, expected); + } + + private boolean isDefaultCorrection(CharSequence typed, CharSequence expected) { + WordComposer word = createWordComposer(typed); + List suggestions = mSuggest.getSuggestions(null, word, false); + return isDefaultSuggestion(suggestions, expected) && mSuggest.hasMinimalCorrection(); + } + + private boolean isASuggestion(CharSequence typed, CharSequence expected) { + WordComposer word = createWordComposer(typed); + List suggestions = mSuggest.getSuggestions(null, word, false); + for (int i = 1; i < suggestions.size(); i++) { + if (TextUtils.equals(suggestions.get(i), expected)) return true; + } + return false; + } + + private boolean isValid(CharSequence typed) { + return mSuggest.isValidWord(typed); + } + + /************************** Tests ************************/ + + /** + * Tests for simple completions of one character. + */ + public void testCompletion1char() { + assertTrue(isDefaultSuggestion("peopl", "people")); + assertTrue(isDefaultSuggestion("abou", "about")); + assertTrue(isDefaultSuggestion("thei", "their")); + } + + /** + * Tests for simple completions of two characters. + */ + public void testCompletion2char() { + assertTrue(isDefaultSuggestion("peop", "people")); + assertTrue(isDefaultSuggestion("calli", "calling")); + assertTrue(isDefaultSuggestion("busine", "business")); + } + + /** + * Tests for proximity errors. + */ + public void testProximityPositive() { + assertTrue(isDefaultSuggestion("peiple", "people")); + assertTrue(isDefaultSuggestion("peoole", "people")); + assertTrue(isDefaultSuggestion("pwpple", "people")); + } + + /** + * Tests for proximity errors - negative, when the error key is not near. + */ + public void testProximityNegative() { + assertFalse(isDefaultSuggestion("arout", "about")); + assertFalse(isDefaultSuggestion("ire", "are")); + } + + /** + * Tests for checking if apostrophes are added automatically. + */ + public void testApostropheInsertion() { + assertTrue(isDefaultSuggestion("im", "I'm")); + assertTrue(isDefaultSuggestion("dont", "don't")); + } + + /** + * Test to make sure apostrophed word is not suggested for an apostrophed word. + */ + public void testApostrophe() { + assertFalse(isDefaultSuggestion("don't", "don't")); + } + + /** + * Tests for suggestion of capitalized version of a word. + */ + public void testCapitalization() { + assertTrue(isDefaultSuggestion("i'm", "I'm")); + assertTrue(isDefaultSuggestion("sunday", "Sunday")); + assertTrue(isDefaultSuggestion("sundat", "Sunday")); + } + + /** + * Tests to see if more than one completion is provided for certain prefixes. + */ + public void testMultipleCompletions() { + assertTrue(isASuggestion("com", "come")); + assertTrue(isASuggestion("com", "company")); + assertTrue(isASuggestion("th", "the")); + assertTrue(isASuggestion("th", "that")); + assertTrue(isASuggestion("th", "this")); + assertTrue(isASuggestion("th", "they")); + } + + /** + * Does the suggestion engine recognize zero frequency words as valid words. + */ + public void testZeroFrequencyAccepted() { + assertTrue(isValid("yikes")); + assertFalse(isValid("yike")); + } + + /** + * Tests to make sure that zero frequency words are not suggested as completions. + */ + public void testZeroFrequencySuggestionsNegative() { + assertFalse(isASuggestion("yike", "yikes")); + assertFalse(isASuggestion("what", "whatcha")); + } + + /** + * Tests to ensure that words with large edit distances are not suggested, in some cases + * and not considered corrections, in some cases. + */ + public void testTooLargeEditDistance() { + assertFalse(isASuggestion("sniyr", "about")); + assertFalse(isDefaultCorrection("rjw", "the")); + } + + /** + * Make sure isValid is case-sensitive. + */ + public void testValidityCaseSensitivity() { + assertTrue(isValid("Sunday")); + assertFalse(isValid("sunday")); + } + + /** + * Are accented forms of words suggested as corrections? + */ + public void testAccents() { + assertTrue(isDefaultCorrection("nino", "ni\u00F1o")); // ni–o + assertTrue(isDefaultCorrection("nimo", "ni\u00F1o")); // ni–o + assertTrue(isDefaultCorrection("maria", "Mar\u00EDa")); // Mar’a + } +}