From 12d63820d459edd71a46fa2495fea98b3c785f2d Mon Sep 17 00:00:00 2001 From: Jatin Matani Date: Thu, 30 Oct 2014 11:33:39 -0700 Subject: [PATCH] Hook for fetching sync content from UserHistoryDict Add API to ExpandableBinaryDictionary to dump content from a given dictionary. We use this for dumping data for sync process. Refactored UserHistoryDictionaryTests to scrap out the util methods for testing. These utility methods would be used for testing sync + user dictionary code in LatinIMEGoogleTests Bug:18106539 Change-Id: I357f9192ea1bd69a526d0b620c25616a2e8e9d5b --- .../latin/ExpandableBinaryDictionary.java | 37 +++++ .../PersonalizationHelper.java | 3 +- .../UserHistoryDictionaryTests.java | 136 ++++------------- .../UserHistoryDictionaryTestsHelper.java | 144 ++++++++++++++++++ 4 files changed, 213 insertions(+), 107 deletions(-) create mode 100644 tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index d9d22e0fc..0b61dc18b 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4; + private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC = + new WordProperty[0] /* default */; + /** The application context. */ protected final Context mContext; @@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } }); } + + /** + * Returns dictionary content required for syncing. + */ + public WordProperty[] getWordPropertiesForSyncing() { + reloadDictionaryIfRequired(); + final AsyncResultHolder result = new AsyncResultHolder<>(); + asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() { + @Override + public void run() { + final ArrayList wordPropertyList = new ArrayList<>(); + final BinaryDictionary binaryDictionary = getBinaryDictionary(); + if (binaryDictionary == null) { + return; + } + int token = 0; + do { + // TODO: We need a new API that returns *new* un-synced data. + final BinaryDictionary.GetNextWordPropertyResult result = + binaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + if (wordProperty == null) { + break; + } + wordPropertyList.add(wordProperty); + token = result.mNextToken; + } while (token != 0); + result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()])); + } + }); + // TODO: Figure out the best timeout duration for this API. + return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC, + TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index 8c5eb0aa7..cefb0ddd5 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -58,8 +58,7 @@ public class PersonalizationHelper { final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { - Log.d(TAG, "Use cached UserHistoryDictionary for " + locale + - " & account" + accountName); + Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr); } dict.reloadDictionaryIfRequired(); return dict; diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index a84df28c9..d83c4a55b 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -25,15 +25,11 @@ import android.util.Log; import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; -import com.android.inputmethod.latin.common.FileUtils; import com.android.inputmethod.latin.settings.LocalSettingsConstants; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; -import java.io.FilenameFilter; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Random; @@ -48,34 +44,13 @@ import javax.annotation.Nullable; public class UserHistoryDictionaryTests extends AndroidTestCase { private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName(); private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000; - private static final String TEST_LOCALE_PREFIX = "test_"; private static final String TEST_ACCOUNT = "account@example.com"; - private static final String[] CHARACTERS = { - "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", - "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" - }; - private int mCurrentTime = 0; private SharedPreferences mPrefs; private String mLastKnownAccount = null; - private void removeAllTestDictFiles() { - final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX); - final String dictName = UserHistoryDictionary.getUserHistoryDictName( - UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); - final File dictFile = ExpandableBinaryDictionary.getDictFile( - mContext, dictName, null /* dictFile */); - final FilenameFilter filenameFilter = new FilenameFilter() { - @Override - public boolean accept(final File dir, final String filename) { - return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX); - } - }; - FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter); - } - private static void printAllFiles(final File dir) { Log.d(TAG, dir.getAbsolutePath()); for (final File file : dir.listFiles()) { @@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } } - private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict, + private static void assertDictionaryExists(final UserHistoryDictionary dict, final File dictFile) { Log.d(TAG, "waiting for writing ..."); dict.waitAllTasksForTests(); @@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Log.e(TAG, "Interrupted during waiting for writing the dict file."); } } - assertTrue("check exisiting of " + dictFile, dictFile.exists()); - FileUtils.deleteRecursively(dictFile); - } - - private static Locale getDummyLocale(final String name) { - return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis()); + assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists()); } @Override @@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { updateAccountName(TEST_ACCOUNT); resetCurrentTimeForTestMode(); - removeAllTestDictFiles(); + UserHistoryDictionaryTestsHelper.removeAllTestDictFiles( + UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext); } @Override protected void tearDown() throws Exception { - removeAllTestDictFiles(); + UserHistoryDictionaryTestsHelper.removeAllTestDictFiles( + UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext); stopTestModeInNativeCode(); // Restore the account that was present before running the test. @@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { return BinaryDictionaryUtils.setCurrentTimeForTest(-1); } - /** - * Generates a random word. - */ - private static String generateWord(final int value) { - final int lengthOfChars = CHARACTERS.length; - final StringBuilder builder = new StringBuilder(); - long lvalue = Math.abs((long)value); - while (lvalue > 0) { - builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); - lvalue /= lengthOfChars; - } - return builder.toString(); - } - - private static List generateWords(final int number, final Random random) { - final HashSet wordSet = new HashSet<>(); - while (wordSet.size() < number) { - wordSet.add(generateWord(random.nextInt())); - } - return new ArrayList<>(wordSet); - } - - private static void addToDict(final UserHistoryDictionary dict, final List words, - final int timestamp) { - NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; - for (final String word : words) { - UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp, - DistracterFilter.EMPTY_DISTRACTER_FILTER); - ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); - } - } - - /** - * @param checkContents if true, checks whether written words are actually in the dictionary - * or not. - */ - private void addAndWriteRandomWords(final UserHistoryDictionary dict, - final int numberOfWords, final Random random, final boolean checkContents) { - final List words = generateWords(numberOfWords, random); - // Add random words to the user history dictionary. - addToDict(dict, words, mCurrentTime); - if (checkContents) { - dict.waitAllTasksForTests(); - for (int i = 0; i < numberOfWords; ++i) { - final String word = words.get(i); - assertTrue(dict.isInDictionary(word)); - } - } - // write to file. - dict.close(); - } - /** * Clear all entries in the user history dictionary. * @param dict the user history dictionary. @@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testRandomWords() { Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); - final Locale dummyLocale = getDummyLocale("random_words"); + final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words"); final String dictName = UserHistoryDictionary.getUserHistoryDictName( UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); final File dictFile = ExpandableBinaryDictionary.getDictFile( mContext, dictName, null /* dictFile */); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( getContext(), dummyLocale, TEST_ACCOUNT); - + clearHistory(dict); final int numberOfWords = 1000; final Random random = new Random(123456); - clearHistory(dict); - addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */); - checkExistenceAndRemoveDictFile(dict, dictFile); + assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords( + dict, numberOfWords, random, true /* checksContents */, mCurrentTime)); + assertDictionaryExists(dict, dictFile); } public void testStressTestForSwitchingLanguagesAndAddingWords() { @@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { - final Locale dummyLocale = getDummyLocale("switching_languages" + i); + final Locale dummyLocale = + UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i); final String dictName = UserHistoryDictionary.getUserHistoryDictName( UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); dictFiles[i] = ExpandableBinaryDictionary.getDictFile( @@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { for (int i = 0; i < numberOfLanguageSwitching; i++) { final int index = i % numberOfLanguages; // Switch to dicts[index]. - addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch, - random, false /* checksContents */); + assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index], + numberOfWordsInsertedForEachLanguageSwitch, + random, + false /* checksContents */, + mCurrentTime)); } final long end = System.currentTimeMillis(); @@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { + (end - start) + " ms"); } finally { for (int i = 0; i < numberOfLanguages; i++) { - checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]); + assertDictionaryExists(dicts[i], dictFiles[i]); } } } public void testAddManyWords() { - final Locale dummyLocale = getDummyLocale("many_random_words"); + final Locale dummyLocale = + UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words"); final String dictName = UserHistoryDictionary.getUserHistoryDictName( UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext()); final File dictFile = ExpandableBinaryDictionary.getDictFile( @@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( getContext(), dummyLocale, TEST_ACCOUNT); clearHistory(dict); - try { - addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */); - } finally { - checkExistenceAndRemoveDictFile(dict, dictFile); - } + assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict, + numberOfWords, random, true /* checksContents */, mCurrentTime)); + assertDictionaryExists(dict, dictFile); } public void testDecaying() { - final Locale dummyLocale = getDummyLocale("decaying"); + final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying"); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary( getContext(), dummyLocale, TEST_ACCOUNT); - final int numberOfWords = 5000; - final Random random = new Random(123456); resetCurrentTimeForTestMode(); clearHistory(dict); - final List words = generateWords(numberOfWords, random); dict.waitAllTasksForTests(); + + final int numberOfWords = 5000; + final Random random = new Random(123456); + final List words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords, + random); NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; for (final String word : words) { UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime, @@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { for (final String word : words) { assertTrue(dict.isInDictionary(word)); } + // Long term decay results in words removed from the dictionary. forcePassingLongTime(); dict.runGCIfRequired(); dict.waitAllTasksForTests(); diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java new file mode 100644 index 000000000..d394c0faa --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import com.android.inputmethod.latin.NgramContext; +import com.android.inputmethod.latin.NgramContext.WordInfo; +import com.android.inputmethod.latin.common.FileUtils; +import com.android.inputmethod.latin.utils.DistracterFilter; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Random; + +/** + * Utility class for helping while running tests involving {@link UserHistoryDictionary}. + */ +public class UserHistoryDictionaryTestsHelper { + + /** + * Locale prefix for generating dummy locales for tests. + */ + public static final String TEST_LOCALE_PREFIX = "test-"; + + /** + * Characters for generating random words. + */ + private static final String[] CHARACTERS = { + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" + }; + + /** + * Remove all the test dictionary files created for the given locale. + */ + public static void removeAllTestDictFiles(final String filter, final Context context) { + final FilenameFilter filenameFilter = new FilenameFilter() { + @Override + public boolean accept(final File dir, final String filename) { + return filename.startsWith(UserHistoryDictionary.NAME + "." + filter); + } + }; + FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter); + } + + /** + * Generates and writes random words to dictionary. Caller can be assured + * that the write tasks would be finished; and its success would be reflected + * in the returned boolean. + * + * @param dict {@link UserHistoryDictionary} to which words should be added. + * @param numberOfWords number of words to be added. + * @param random helps generate random words. + * @param checkContents if true, checks whether written words are actually in the dictionary. + * @param currentTime timestamp that would be used for adding the words. + * @returns true if all words have been written to dictionary successfully. + */ + public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict, + final int numberOfWords, final Random random, final boolean checkContents, + final int currentTime) { + final List words = generateWords(numberOfWords, random); + // Add random words to the user history dictionary. + addWordsToDictionary(dict, words, currentTime); + boolean success = true; + if (checkContents) { + dict.waitAllTasksForTests(); + for (int i = 0; i < numberOfWords; ++i) { + final String word = words.get(i); + if (!dict.isInDictionary(word)) { + success = false; + break; + } + } + } + // write to file. + dict.close(); + dict.waitAllTasksForTests(); + return success; + } + + private static void addWordsToDictionary(final UserHistoryDictionary dict, + final List words, final int timestamp) { + NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; + for (final String word : words) { + UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp, + DistracterFilter.EMPTY_DISTRACTER_FILTER); + ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); + } + } + + /** + * Creates unique test locale for using within tests. + */ + public static Locale getDummyLocale(final String name) { + return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis()); + } + + /** + * Generates random words. + * + * @param numberOfWords number of words to generate. + * @param random salt used for generating random words. + */ + public static List generateWords(final int numberOfWords, final Random random) { + final HashSet wordSet = new HashSet<>(); + while (wordSet.size() < numberOfWords) { + wordSet.add(generateWord(random.nextInt())); + } + return new ArrayList<>(wordSet); + } + + /** + * Generates a random word. + */ + private static String generateWord(final int value) { + final int lengthOfChars = CHARACTERS.length; + final StringBuilder builder = new StringBuilder(); + long lvalue = Math.abs((long)value); + while (lvalue > 0) { + builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]); + lvalue /= lengthOfChars; + } + return builder.toString(); + } +}