Hook for fetching sync content from UserHistoryDict

Add API to ExpandableBinaryDictionary to dump content
from a given dictionary. We use this for dumping data
for sync process.

Refactored UserHistoryDictionaryTests to scrap out the util
methods for testing. These utility methods would be used
for testing sync + user dictionary code in LatinIMEGoogleTests

Bug:18106539
Change-Id: I357f9192ea1bd69a526d0b620c25616a2e8e9d5b

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
 
     private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;
 
+    private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =
+            new WordProperty[0] /* default */;
+
     /** The application context. */
     protected final Context mContext;
 
@@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             }
         });
     }
+
+    /**
+     * Returns dictionary content required for syncing.
+     */
+    public WordProperty[] getWordPropertiesForSyncing() {
+        reloadDictionaryIfRequired();
+        final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();
+        asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {
+            @Override
+            public void run() {
+                final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();
+                final BinaryDictionary binaryDictionary = getBinaryDictionary();
+                if (binaryDictionary == null) {
+                    return;
+                }
+                int token = 0;
+                do {
+                    // TODO: We need a new API that returns *new* un-synced data.
+                    final BinaryDictionary.GetNextWordPropertyResult result =
+                            binaryDictionary.getNextWordProperty(token);
+                    final WordProperty wordProperty = result.mWordProperty;
+                    if (wordProperty == null) {
+                        break;
+                    }
+                    wordPropertyList.add(wordProperty);
+                    token = result.mNextToken;
+                } while (token != 0);
+                result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));
+            }
+        });
+        // TODO: Figure out the best timeout duration for this API.
+        return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,
+                TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
+    }
 }

java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java

@@ -58,8 +58,7 @@ public class PersonalizationHelper {
                 final UserHistoryDictionary dict = ref == null ? null : ref.get();
                 if (dict != null) {
                     if (DEBUG) {
-                        Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +
+                        Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);
-                                " & account" + accountName);
                     }
                     dict.reloadDictionaryIfRequired();
                     return dict;

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java

@@ -25,15 +25,11 @@ import android.util.Log;
 import com.android.inputmethod.latin.ExpandableBinaryDictionary;
 import com.android.inputmethod.latin.NgramContext;
 import com.android.inputmethod.latin.NgramContext.WordInfo;
-import com.android.inputmethod.latin.common.FileUtils;
 import com.android.inputmethod.latin.settings.LocalSettingsConstants;
 import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
 import com.android.inputmethod.latin.utils.DistracterFilter;
 
 import java.io.File;
-import java.io.FilenameFilter;
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Random;
@@ -48,34 +44,13 @@ import javax.annotation.Nullable;
 public class UserHistoryDictionaryTests extends AndroidTestCase {
     private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
     private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
-    private static final String TEST_LOCALE_PREFIX = "test_";
     private static final String TEST_ACCOUNT = "account@example.com";
 
-    private static final String[] CHARACTERS = {
-        "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
-        "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
-    };
-
     private int mCurrentTime = 0;
 
     private SharedPreferences mPrefs;
     private String mLastKnownAccount = null;
 
-    private void removeAllTestDictFiles() {
-        final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
-        final String dictName = UserHistoryDictionary.getUserHistoryDictName(
-                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
-        final File dictFile = ExpandableBinaryDictionary.getDictFile(
-                mContext, dictName, null /* dictFile */);
-        final FilenameFilter filenameFilter = new FilenameFilter() {
-            @Override
-            public boolean accept(final File dir, final String filename) {
-                return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
-            }
-        };
-        FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
-    }
-
     private static void printAllFiles(final File dir) {
         Log.d(TAG, dir.getAbsolutePath());
         for (final File file : dir.listFiles()) {
@@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
         }
     }
 
-    private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
+    private static void assertDictionaryExists(final UserHistoryDictionary dict,
             final File dictFile) {
         Log.d(TAG, "waiting for writing ...");
         dict.waitAllTasksForTests();
@@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
                 Log.e(TAG, "Interrupted during waiting for writing the dict file.");
             }
         }
-        assertTrue("check exisiting of " + dictFile, dictFile.exists());
+        assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());
-        FileUtils.deleteRecursively(dictFile);
-    }
-
-    private static Locale getDummyLocale(final String name) {
-        return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
     }
 
     @Override
@@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
         updateAccountName(TEST_ACCOUNT);
 
         resetCurrentTimeForTestMode();
-        removeAllTestDictFiles();
+        UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
+                UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
     }
 
     @Override
     protected void tearDown() throws Exception {
-        removeAllTestDictFiles();
+        UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
+                UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
         stopTestModeInNativeCode();
 
         // Restore the account that was present before running the test.
@@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
         return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
     }
 
-    /**
-     * Generates a random word.
-     */
-    private static String generateWord(final int value) {
-        final int lengthOfChars = CHARACTERS.length;
-        final StringBuilder builder = new StringBuilder();
-        long lvalue = Math.abs((long)value);
-        while (lvalue > 0) {
-            builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
-            lvalue /= lengthOfChars;
-        }
-        return builder.toString();
-    }
-
-    private static List<String> generateWords(final int number, final Random random) {
-        final HashSet<String> wordSet = new HashSet<>();
-        while (wordSet.size() < number) {
-            wordSet.add(generateWord(random.nextInt()));
-        }
-        return new ArrayList<>(wordSet);
-    }
-
-    private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
-            final int timestamp) {
-        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
-        for (final String word : words) {
-            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
-                    DistracterFilter.EMPTY_DISTRACTER_FILTER);
-            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
-        }
-    }
-
-    /**
-     * @param checkContents if true, checks whether written words are actually in the dictionary
-     * or not.
-     */
-    private void addAndWriteRandomWords(final UserHistoryDictionary dict,
-            final int numberOfWords, final Random random, final boolean checkContents) {
-        final List<String> words = generateWords(numberOfWords, random);
-        // Add random words to the user history dictionary.
-        addToDict(dict, words, mCurrentTime);
-        if (checkContents) {
-            dict.waitAllTasksForTests();
-            for (int i = 0; i < numberOfWords; ++i) {
-                final String word = words.get(i);
-                assertTrue(dict.isInDictionary(word));
-            }
-        }
-        // write to file.
-        dict.close();
-    }
-
     /**
      * Clear all entries in the user history dictionary.
      * @param dict the user history dictionary.
@@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
     public void testRandomWords() {
         Log.d(TAG, "This test can be used for profiling.");
         Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
-        final Locale dummyLocale = getDummyLocale("random_words");
+        final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");
         final String dictName = UserHistoryDictionary.getUserHistoryDictName(
                 UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
         final File dictFile = ExpandableBinaryDictionary.getDictFile(
                 mContext, dictName, null /* dictFile */);
         final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
                 getContext(), dummyLocale, TEST_ACCOUNT);
-
+        clearHistory(dict);
         final int numberOfWords = 1000;
         final Random random = new Random(123456);
-        clearHistory(dict);
+        assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(
-        addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
+                dict, numberOfWords, random, true /* checksContents */, mCurrentTime));
-        checkExistenceAndRemoveDictFile(dict, dictFile);
+        assertDictionaryExists(dict, dictFile);
     }
 
     public void testStressTestForSwitchingLanguagesAndAddingWords() {
@@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
 
             // Create filename suffixes for this test.
             for (int i = 0; i < numberOfLanguages; i++) {
-                final Locale dummyLocale = getDummyLocale("switching_languages" + i);
+                final Locale dummyLocale =
+                        UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);
                 final String dictName = UserHistoryDictionary.getUserHistoryDictName(
                         UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
                 dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
@@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
             for (int i = 0; i < numberOfLanguageSwitching; i++) {
                 final int index = i % numberOfLanguages;
                 // Switch to dicts[index].
-                addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
+                assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],
-                        random, false /* checksContents */);
+                        numberOfWordsInsertedForEachLanguageSwitch,
+                        random,
+                        false /* checksContents */,
+                        mCurrentTime));
             }
 
             final long end = System.currentTimeMillis();
@@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
                     + (end - start) + " ms");
         } finally {
             for (int i = 0; i < numberOfLanguages; i++) {
-                checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
+                assertDictionaryExists(dicts[i], dictFiles[i]);
             }
         }
     }
 
     public void testAddManyWords() {
-        final Locale dummyLocale = getDummyLocale("many_random_words");
+        final Locale dummyLocale =
+                UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");
         final String dictName = UserHistoryDictionary.getUserHistoryDictName(
                 UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
         final File dictFile = ExpandableBinaryDictionary.getDictFile(
@@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
         final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
                 getContext(), dummyLocale, TEST_ACCOUNT);
         clearHistory(dict);
-        try {
+        assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,
-            addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
+                numberOfWords, random, true /* checksContents */, mCurrentTime));
-        } finally {
+        assertDictionaryExists(dict, dictFile);
-            checkExistenceAndRemoveDictFile(dict, dictFile);
-        }
     }
 
     public void testDecaying() {
-        final Locale dummyLocale = getDummyLocale("decaying");
+        final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
         final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
                 getContext(), dummyLocale, TEST_ACCOUNT);
-        final int numberOfWords = 5000;
-        final Random random = new Random(123456);
         resetCurrentTimeForTestMode();
         clearHistory(dict);
-        final List<String> words = generateWords(numberOfWords, random);
         dict.waitAllTasksForTests();
+
+        final int numberOfWords = 5000;
+        final Random random = new Random(123456);
+        final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
+                random);
         NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
         for (final String word : words) {
             UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
@@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
         for (final String word : words) {
             assertTrue(dict.isInDictionary(word));
         }
+        // Long term decay results in words removed from the dictionary.
         forcePassingLongTime();
         dict.runGCIfRequired();
         dict.waitAllTasksForTests();

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.personalization;
+
+import android.content.Context;
+
+import com.android.inputmethod.latin.NgramContext;
+import com.android.inputmethod.latin.NgramContext.WordInfo;
+import com.android.inputmethod.latin.common.FileUtils;
+import com.android.inputmethod.latin.utils.DistracterFilter;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Random;
+
+/**
+ * Utility class for helping while running tests involving {@link UserHistoryDictionary}.
+ */
+public class UserHistoryDictionaryTestsHelper {
+
+    /**
+     * Locale prefix for generating dummy locales for tests.
+     */
+    public static final String TEST_LOCALE_PREFIX = "test-";
+
+    /**
+     * Characters for generating random words.
+     */
+    private static final String[] CHARACTERS = {
+        "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
+        "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
+    };
+
+    /**
+     * Remove all the test dictionary files created for the given locale.
+     */
+    public static void removeAllTestDictFiles(final String filter, final Context context) {
+        final FilenameFilter filenameFilter = new FilenameFilter() {
+            @Override
+            public boolean accept(final File dir, final String filename) {
+                return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);
+            }
+        };
+        FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);
+    }
+
+    /**
+     * Generates and writes random words to dictionary. Caller can be assured
+     * that the write tasks would be finished; and its success would be reflected
+     * in the returned boolean.
+     *
+     * @param dict {@link UserHistoryDictionary} to which words should be added.
+     * @param numberOfWords number of words to be added.
+     * @param random helps generate random words.
+     * @param checkContents if true, checks whether written words are actually in the dictionary.
+     * @param currentTime timestamp that would be used for adding the words.
+     * @returns true if all words have been written to dictionary successfully.
+     */
+    public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,
+            final int numberOfWords, final Random random, final boolean checkContents,
+            final int currentTime) {
+        final List<String> words = generateWords(numberOfWords, random);
+        // Add random words to the user history dictionary.
+        addWordsToDictionary(dict, words, currentTime);
+        boolean success = true;
+        if (checkContents) {
+            dict.waitAllTasksForTests();
+            for (int i = 0; i < numberOfWords; ++i) {
+                final String word = words.get(i);
+                if (!dict.isInDictionary(word)) {
+                    success = false;
+                    break;
+                }
+            }
+        }
+        // write to file.
+        dict.close();
+        dict.waitAllTasksForTests();
+        return success;
+    }
+
+    private static void addWordsToDictionary(final UserHistoryDictionary dict,
+            final List<String> words, final int timestamp) {
+        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
+        for (final String word : words) {
+            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
+                    DistracterFilter.EMPTY_DISTRACTER_FILTER);
+            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
+        }
+    }
+
+    /**
+     * Creates unique test locale for using within tests.
+     */
+    public static Locale getDummyLocale(final String name) {
+        return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
+    }
+
+    /**
+     * Generates random words.
+     *
+     * @param numberOfWords number of words to generate.
+     * @param random salt used for generating random words.
+     */
+    public static List<String> generateWords(final int numberOfWords, final Random random) {
+        final HashSet<String> wordSet = new HashSet<>();
+        while (wordSet.size() < numberOfWords) {
+            wordSet.add(generateWord(random.nextInt()));
+        }
+        return new ArrayList<>(wordSet);
+    }
+
+    /**
+     * Generates a random word.
+     */
+    private static String generateWord(final int value) {
+        final int lengthOfChars = CHARACTERS.length;
+        final StringBuilder builder = new StringBuilder();
+        long lvalue = Math.abs((long)value);
+        while (lvalue > 0) {
+            builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
+            lvalue /= lengthOfChars;
+        }
+        return builder.toString();
+    }
+}