Extend jni interface to get bigrams in WordProperty.

Bug: 12810574
Change-Id: Ia4b88d02ea8790a5c47d32376cc0b84c3e071ddd

java/src/com/android/inputmethod/latin/BinaryDictionary.java

@@ -68,11 +68,12 @@ public final class BinaryDictionary extends Dictionary {
     private static final int FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX = 2;
     private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3;
 
-    // Format to get unigram historical info from native side via getWordPropertyNative().
+    // Format to get probability and historical info from native side via getWordPropertyNative().
-    private static final int FORMAT_WORD_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT = 3;
+    public static final int FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT = 4;
-    private static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 0;
+    public static final int FORMAT_WORD_PROPERTY_PROBABILITY_INDEX = 0;
-    private static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 1;
+    public static final int FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX = 1;
-    private static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 2;
+    public static final int FORMAT_WORD_PROPERTY_LEVEL_INDEX = 2;
+    public static final int FORMAT_WORD_PROPERTY_COUNT_INDEX = 3;
 
     private long mNativeDict;
     private final Locale mLocale;
@@ -144,9 +145,9 @@ public final class BinaryDictionary extends Dictionary {
     private static native int getProbabilityNative(long dict, int[] word);
     private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
     private static native void getWordPropertyNative(long dict, int[] word,
-            int[] outCodePoints, boolean[] outFlags, int[] outProbability,
+            int[] outCodePoints, boolean[] outFlags, int[] outProbabilityInfo,
-            int[] outHistoricalInfo, ArrayList<int[]> outShortcutTargets,
+            ArrayList<int[]> outBigramTargets, ArrayList<int[]> outBigramProbabilityInfo,
-            ArrayList<Integer> outShortcutProbabilities);
+            ArrayList<int[]> outShortcutTargets, ArrayList<Integer> outShortcutProbabilities);
     private static native int getSuggestionsNative(long dict, long proximityInfo,
             long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times,
             int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint,
@@ -313,22 +314,22 @@ public final class BinaryDictionary extends Dictionary {
         final int[] codePoints = StringUtils.toCodePointArray(word);
         final int[] outCodePoints = new int[MAX_WORD_LENGTH];
         final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT];
-        final int[] outProbability = new int[1];
+        final int[] outProbabilityInfo =
-        final int[] outHistoricalInfo =
+                new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT];
-                new int[FORMAT_WORD_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT];
+        final ArrayList<int[]> outBigramTargets = CollectionUtils.newArrayList();
+        final ArrayList<int[]> outBigramProbabilityInfo = CollectionUtils.newArrayList();
         final ArrayList<int[]> outShortcutTargets = CollectionUtils.newArrayList();
         final ArrayList<Integer> outShortcutProbabilities = CollectionUtils.newArrayList();
-        getWordPropertyNative(mNativeDict, codePoints, outCodePoints, outFlags, outProbability,
+        getWordPropertyNative(mNativeDict, codePoints, outCodePoints, outFlags, outProbabilityInfo,
-                outHistoricalInfo, outShortcutTargets, outShortcutProbabilities);
+                outBigramTargets, outBigramProbabilityInfo, outShortcutTargets,
+                outShortcutProbabilities);
         return new WordProperty(codePoints,
                 outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX],
                 outFlags[FORMAT_WORD_PROPERTY_IS_BLACKLISTED_INDEX],
                 outFlags[FORMAT_WORD_PROPERTY_HAS_BIGRAMS_INDEX],
-                outFlags[FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX], outProbability[0],
+                outFlags[FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX], outProbabilityInfo,
-                outHistoricalInfo[FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX],
+                outBigramTargets, outBigramProbabilityInfo, outShortcutTargets,
-                outHistoricalInfo[FORMAT_WORD_PROPERTY_LEVEL_INDEX],
+                outShortcutProbabilities);
-                outHistoricalInfo[FORMAT_WORD_PROPERTY_COUNT_INDEX],
-                outShortcutTargets, outShortcutProbabilities);
     }
 
     // Add a unigram entry to binary dictionary with unigram attributes in native code.

java/src/com/android/inputmethod/latin/utils/WordProperty.java

@@ -32,14 +32,29 @@ public class WordProperty {
     public final boolean mIsBlacklisted;
     public final boolean mHasBigrams;
     public final boolean mHasShortcuts;
+    public final ProbabilityInfo mProbabilityInfo;
+    public final ArrayList<WeightedString> mBigramTargets = CollectionUtils.newArrayList();
+    public final ArrayList<ProbabilityInfo> mBigramProbabilityInfo = CollectionUtils.newArrayList();
+    public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList();
+
+    // TODO: Use this kind of Probability class for dictionary read/write code under the makedict
+    // package.
+    public static final class ProbabilityInfo {
         public final int mProbability;
-    // mTimestamp, mLevel and mCount are historical info. These values are depend on the
+        // wTimestamp, mLevel and mCount are historical info. These values are depend on the
         // implementation in native code; thus, we must not use them and have any assumptions about
         // them except for tests.
         public final int mTimestamp;
         public final int mLevel;
         public final int mCount;
-    public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList();
+
+        public ProbabilityInfo(final int[] probabilityInfo) {
+            mProbability = probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX];
+            mTimestamp = probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX];
+            mLevel = probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX];
+            mCount = probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX];
+        }
+    }
 
     private static int getCodePointCount(final int[] codePoints) {
         for (int i = 0; i < codePoints.length; i++) {
@@ -53,18 +68,29 @@ public class WordProperty {
     // This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY.
     public WordProperty(final int[] codePoints, final boolean isNotAWord,
             final boolean isBlacklisted, final boolean hasBigram,
-            final boolean hasShortcuts, final int probability, final int timestamp,
+            final boolean hasShortcuts, final int[] probabilityInfo,
-            final int level, final int count, final ArrayList<int[]> shortcutTargets,
+            final ArrayList<int[]> bigramTargets, final ArrayList<int[]> bigramProbabilityInfo,
+            final ArrayList<int[]> shortcutTargets,
             final ArrayList<Integer> shortcutProbabilities) {
         mCodePoints = new String(codePoints, 0 /* offset */, getCodePointCount(codePoints));
         mIsNotAWord = isNotAWord;
         mIsBlacklisted = isBlacklisted;
         mHasBigrams = hasBigram;
         mHasShortcuts = hasShortcuts;
-        mProbability = probability;
+        mProbabilityInfo = new ProbabilityInfo(probabilityInfo);
-        mTimestamp = timestamp;
+
-        mLevel = level;
+        final int bigramTargetCount = bigramTargets.size();
-        mCount = count;
+        for (int i = 0; i < bigramTargetCount; i++) {
+            final int[] bigramTargetCodePointArray = bigramTargets.get(i);
+            final String bigramTargetString = new String(bigramTargetCodePointArray,
+                    0 /* offset */, getCodePointCount(bigramTargetCodePointArray));
+            final ProbabilityInfo bigramProbability =
+                    new ProbabilityInfo(bigramProbabilityInfo.get(i));
+            mBigramTargets.add(
+                    new WeightedString(bigramTargetString, bigramProbability.mProbability));
+            mBigramProbabilityInfo.add(bigramProbability);
+        }
+
         final int shortcutTargetCount = shortcutTargets.size();
         for (int i = 0; i < shortcutTargetCount; i++) {
             final int[] shortcutTargetCodePointArray = shortcutTargets.get(i);
@@ -77,6 +103,6 @@ public class WordProperty {
 
     @UsedForTesting
     public boolean isValid() {
-        return mProbability != BinaryDictionary.NOT_A_PROBABILITY;
+        return mProbabilityInfo.mProbability != BinaryDictionary.NOT_A_PROBABILITY;
     }
 }

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

@@ -262,16 +262,17 @@ static jint latinime_BinaryDictionary_getBigramProbability(JNIEnv *env, jclass c
 
 static void latinime_BinaryDictionary_getWordProperty(JNIEnv *env, jclass clazz,
         jlong dict, jintArray word, jintArray outCodePoints, jbooleanArray outFlags,
-        jintArray outProbability, jintArray outHistoricalInfo, jobject outShortcutTargets,
+        jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilityInfo,
-        jobject outShortcutProbabilities) {
+        jobject outShortcutTargets, jobject outShortcutProbabilities) {
     Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
     if (!dictionary) return;
     const jsize wordLength = env->GetArrayLength(word);
     int wordCodePoints[wordLength];
     env->GetIntArrayRegion(word, 0, wordLength, wordCodePoints);
     const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength);
-    wordProperty.outputProperties(env, outCodePoints, outFlags, outProbability,
+    wordProperty.outputProperties(env, outCodePoints, outFlags, outProbabilityInfo,
-            outHistoricalInfo, outShortcutTargets, outShortcutProbabilities);
+            outBigramTargets, outBigramProbabilityInfo, outShortcutTargets,
+            outShortcutProbabilities);
 }
 
 static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jclass clazz,
@@ -521,7 +522,8 @@ static const JNINativeMethod sMethods[] = {
     },
     {
         const_cast<char *>("getWordPropertyNative"),
-        const_cast<char *>("(J[I[I[Z[I[ILjava/util/ArrayList;Ljava/util/ArrayList;)V"),
+        const_cast<char *>("(J[I[I[Z[ILjava/util/ArrayList;Ljava/util/ArrayList;"
+                "Ljava/util/ArrayList;Ljava/util/ArrayList;)V"),
         reinterpret_cast<void *>(latinime_BinaryDictionary_getWordProperty)
     },
     {

native/jni/src/suggest/core/dictionary/word_property.cpp

@@ -19,20 +19,23 @@
 namespace latinime {
 
 void WordProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
-        jbooleanArray outFlags, jintArray outProbability, jintArray outHistoricalInfo,
+        jbooleanArray outFlags, jintArray outProbabilityInfo, jobject outBigramTargets,
-        jobject outShortcutTargets, jobject outShortcutProbabilities) const {
+        jobject outBigramProbabilities, jobject outShortcutTargets,
+        jobject outShortcutProbabilities) const {
     env->SetIntArrayRegion(outCodePoints, 0 /* start */, mCodePoints.size(), &mCodePoints[0]);
     jboolean flags[] = {mIsNotAWord, mIsBlacklisted, mHasBigrams, mHasShortcuts};
     env->SetBooleanArrayRegion(outFlags, 0 /* start */, NELEMS(flags), flags);
-    env->SetIntArrayRegion(outProbability, 0 /* start */, 1 /* len */, &mProbability);
+    int probabilityInfo[] = {mProbability, mTimestamp, mLevel, mCount};
-    int historicalInfo[] = {mTimestamp, mLevel, mCount};
+    env->SetIntArrayRegion(outProbabilityInfo, 0 /* start */, NELEMS(probabilityInfo),
-    env->SetIntArrayRegion(outHistoricalInfo, 0 /* start */, NELEMS(historicalInfo),
+            probabilityInfo);
-            historicalInfo);
 
     jclass integerClass = env->FindClass("java/lang/Integer");
     jmethodID intToIntegerConstructorId = env->GetMethodID(integerClass, "<init>", "(I)V");
     jclass arrayListClass = env->FindClass("java/util/ArrayList");
     jmethodID addMethodId = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
+
+    // TODO: Output bigrams.
+    // Output shortcuts.
     const int shortcutTargetCount = mShortcuts.size();
     for (int i = 0; i < shortcutTargetCount; ++i) {
         const std::vector<int> *const targetCodePoints = mShortcuts[i].getTargetCodePoints();

native/jni/src/suggest/core/dictionary/word_property.h

@@ -78,8 +78,8 @@ class WordProperty {
               mShortcuts(*shortcuts) {}
 
     void outputProperties(JNIEnv *const env, jintArray outCodePoints, jbooleanArray outFlags,
-            jintArray outProbability, jintArray outHistoricalInfo, jobject outShortcutTargets,
+            jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilities,
-            jobject outShortcutProbabilities) const;
+            jobject outShortcutTargets, jobject outShortcutProbabilities) const;
 
  private:
     DISALLOW_ASSIGNMENT_OPERATOR(WordProperty);

tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java

@@ -871,11 +871,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
         }
     }
 
-    public void testGetUnigramProperties() {
+    public void testGetWordProperties() {
-        testGetUnigramProperties(FormatSpec.VERSION4);
+        testGetWordProperties(FormatSpec.VERSION4);
     }
 
-    private void testGetUnigramProperties(final int formatVersion) {
+    private void testGetWordProperties(final int formatVersion) {
         final long seed = System.currentTimeMillis();
         final Random random = new Random(seed);
         final int ITERATION_COUNT = 1000;
@@ -892,8 +892,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
                 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
                 Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
 
-        final WordProperty invalidUnigramProperty = binaryDictionary.getWordProperty("dummyWord");
+        final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord");
-        assertFalse(invalidUnigramProperty.isValid());
+        assertFalse(invalidWordProperty.isValid());
 
         for (int i = 0; i < ITERATION_COUNT; i++) {
             final String word = CodePointUtils.generateWord(random, codePointSet);
@@ -904,15 +904,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
             binaryDictionary.addUnigramWord(word, unigramProbability,
                     null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY,
                     isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
-            final WordProperty unigramProperty = binaryDictionary.getWordProperty(word);
+            final WordProperty wordProperty = binaryDictionary.getWordProperty(word);
-            assertEquals(word, unigramProperty.mCodePoints);
+            assertEquals(word, wordProperty.mCodePoints);
-            assertTrue(unigramProperty.isValid());
+            assertTrue(wordProperty.isValid());
-            assertEquals(isNotAWord, unigramProperty.mIsNotAWord);
+            assertEquals(isNotAWord, wordProperty.mIsNotAWord);
-            assertEquals(isBlacklisted, unigramProperty.mIsBlacklisted);
+            assertEquals(isBlacklisted, wordProperty.mIsBlacklisted);
-            assertEquals(false, unigramProperty.mHasBigrams);
+            assertEquals(false, wordProperty.mHasBigrams);
-            assertEquals(false, unigramProperty.mHasShortcuts);
+            assertEquals(false, wordProperty.mHasShortcuts);
-            assertEquals(unigramProbability, unigramProperty.mProbability);
+            assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability);
-            assertTrue(unigramProperty.mShortcutTargets.isEmpty());
+            assertTrue(wordProperty.mShortcutTargets.isEmpty());
         }
     }
 
@@ -936,28 +936,28 @@ public class BinaryDictionaryTests extends AndroidTestCase {
         binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
                 shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
                 0 /* timestamp */);
-        WordProperty unigramProperty = binaryDictionary.getWordProperty("aaa");
+        WordProperty wordProperty = binaryDictionary.getWordProperty("aaa");
-        assertEquals(1, unigramProperty.mShortcutTargets.size());
+        assertEquals(1, wordProperty.mShortcutTargets.size());
-        assertEquals("zzz", unigramProperty.mShortcutTargets.get(0).mWord);
+        assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
-        assertEquals(shortcutProbability, unigramProperty.mShortcutTargets.get(0).mFrequency);
+        assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).mFrequency);
         final int updatedShortcutProbability = 2;
         binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
                 updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
                 0 /* timestamp */);
-        unigramProperty = binaryDictionary.getWordProperty("aaa");
+        wordProperty = binaryDictionary.getWordProperty("aaa");
-        assertEquals(1, unigramProperty.mShortcutTargets.size());
+        assertEquals(1, wordProperty.mShortcutTargets.size());
-        assertEquals("zzz", unigramProperty.mShortcutTargets.get(0).mWord);
+        assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
         assertEquals(updatedShortcutProbability,
-                unigramProperty.mShortcutTargets.get(0).mFrequency);
+                wordProperty.mShortcutTargets.get(0).mFrequency);
         binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy",
                 shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
                 0 /* timestamp */);
         final HashMap<String, Integer> shortcutTargets = new HashMap<String, Integer>();
         shortcutTargets.put("zzz", updatedShortcutProbability);
         shortcutTargets.put("yyy", shortcutProbability);
-        unigramProperty = binaryDictionary.getWordProperty("aaa");
+        wordProperty = binaryDictionary.getWordProperty("aaa");
-        assertEquals(2, unigramProperty.mShortcutTargets.size());
+        assertEquals(2, wordProperty.mShortcutTargets.size());
-        for (WeightedString shortcutTarget : unigramProperty.mShortcutTargets) {
+        for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
             assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
             assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
             shortcutTargets.remove(shortcutTarget.mWord);
@@ -965,9 +965,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
         shortcutTargets.put("zzz", updatedShortcutProbability);
         shortcutTargets.put("yyy", shortcutProbability);
         binaryDictionary.flushWithGC();
-        unigramProperty = binaryDictionary.getWordProperty("aaa");
+        wordProperty = binaryDictionary.getWordProperty("aaa");
-        assertEquals(2, unigramProperty.mShortcutTargets.size());
+        assertEquals(2, wordProperty.mShortcutTargets.size());
-        for (WeightedString shortcutTarget : unigramProperty.mShortcutTargets) {
+        for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
             assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
             assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
             shortcutTargets.remove(shortcutTarget.mWord);
@@ -1034,14 +1034,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
         }
 
         for (final String word : words) {
-            final WordProperty unigramProperty = binaryDictionary.getWordProperty(word);
+            final WordProperty wordProperty = binaryDictionary.getWordProperty(word);
-            assertEquals((int)unigramProbabilities.get(word), unigramProperty.mProbability);
+            assertEquals((int)unigramProbabilities.get(word),
+                    wordProperty.mProbabilityInfo.mProbability);
             if (!shortcutTargets.containsKey(word)) {
                 // The word does not have shortcut targets.
                 continue;
             }
-            assertEquals(shortcutTargets.get(word).size(), unigramProperty.mShortcutTargets.size());
+            assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size());
-            for (final WeightedString shortcutTarget : unigramProperty.mShortcutTargets) {
+            for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
                 final String targetCodePonts = shortcutTarget.mWord;
                 assertEquals((int)shortcutTargets.get(word).get(targetCodePonts),
                         shortcutTarget.mFrequency);