Hook for fetching sync content from UserHistoryDict
Add API to ExpandableBinaryDictionary to dump content from a given dictionary. We use this for dumping data for sync process. Refactored UserHistoryDictionaryTests to scrap out the util methods for testing. These utility methods would be used for testing sync + user dictionary code in LatinIMEGoogleTests Bug:18106539 Change-Id: I357f9192ea1bd69a526d0b620c25616a2e8e9d5b
This commit is contained in:
Jatin Matani
parent
4195567d24
commit
12d63820d4
4 changed files with 213 additions and 107 deletions
|
|
@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
|||
|
||||
private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;
|
||||
|
||||
private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =
|
||||
new WordProperty[0] /* default */;
|
||||
|
||||
/** The application context. */
|
||||
protected final Context mContext;
|
||||
|
||||
|
|
@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns dictionary content required for syncing.
|
||||
*/
|
||||
public WordProperty[] getWordPropertiesForSyncing() {
|
||||
reloadDictionaryIfRequired();
|
||||
final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();
|
||||
asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();
|
||||
final BinaryDictionary binaryDictionary = getBinaryDictionary();
|
||||
if (binaryDictionary == null) {
|
||||
return;
|
||||
}
|
||||
int token = 0;
|
||||
do {
|
||||
// TODO: We need a new API that returns *new* un-synced data.
|
||||
final BinaryDictionary.GetNextWordPropertyResult result =
|
||||
binaryDictionary.getNextWordProperty(token);
|
||||
final WordProperty wordProperty = result.mWordProperty;
|
||||
if (wordProperty == null) {
|
||||
break;
|
||||
}
|
||||
wordPropertyList.add(wordProperty);
|
||||
token = result.mNextToken;
|
||||
} while (token != 0);
|
||||
result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));
|
||||
}
|
||||
});
|
||||
// TODO: Figure out the best timeout duration for this API.
|
||||
return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,
|
||||
TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,8 +58,7 @@ public class PersonalizationHelper {
|
|||
final UserHistoryDictionary dict = ref == null ? null : ref.get();
|
||||
if (dict != null) {
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +
|
||||
" & account" + accountName);
|
||||
Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);
|
||||
}
|
||||
dict.reloadDictionaryIfRequired();
|
||||
return dict;
|
||||
|
|
|
|||
|
|
@ -25,15 +25,11 @@ import android.util.Log;
|
|||
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
|
||||
import com.android.inputmethod.latin.NgramContext;
|
||||
import com.android.inputmethod.latin.NgramContext.WordInfo;
|
||||
import com.android.inputmethod.latin.common.FileUtils;
|
||||
import com.android.inputmethod.latin.settings.LocalSettingsConstants;
|
||||
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
|
||||
import com.android.inputmethod.latin.utils.DistracterFilter;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FilenameFilter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Random;
|
||||
|
|
@ -48,34 +44,13 @@ import javax.annotation.Nullable;
|
|||
public class UserHistoryDictionaryTests extends AndroidTestCase {
|
||||
private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
|
||||
private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
|
||||
private static final String TEST_LOCALE_PREFIX = "test_";
|
||||
private static final String TEST_ACCOUNT = "account@example.com";
|
||||
|
||||
private static final String[] CHARACTERS = {
|
||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
};
|
||||
|
||||
private int mCurrentTime = 0;
|
||||
|
||||
private SharedPreferences mPrefs;
|
||||
private String mLastKnownAccount = null;
|
||||
|
||||
private void removeAllTestDictFiles() {
|
||||
final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
|
||||
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
|
||||
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
|
||||
final File dictFile = ExpandableBinaryDictionary.getDictFile(
|
||||
mContext, dictName, null /* dictFile */);
|
||||
final FilenameFilter filenameFilter = new FilenameFilter() {
|
||||
@Override
|
||||
public boolean accept(final File dir, final String filename) {
|
||||
return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
|
||||
}
|
||||
};
|
||||
FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
|
||||
}
|
||||
|
||||
private static void printAllFiles(final File dir) {
|
||||
Log.d(TAG, dir.getAbsolutePath());
|
||||
for (final File file : dir.listFiles()) {
|
||||
|
|
@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
|
||||
private static void assertDictionaryExists(final UserHistoryDictionary dict,
|
||||
final File dictFile) {
|
||||
Log.d(TAG, "waiting for writing ...");
|
||||
dict.waitAllTasksForTests();
|
||||
|
|
@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
Log.e(TAG, "Interrupted during waiting for writing the dict file.");
|
||||
}
|
||||
}
|
||||
assertTrue("check exisiting of " + dictFile, dictFile.exists());
|
||||
FileUtils.deleteRecursively(dictFile);
|
||||
}
|
||||
|
||||
private static Locale getDummyLocale(final String name) {
|
||||
return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
|
||||
assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
updateAccountName(TEST_ACCOUNT);
|
||||
|
||||
resetCurrentTimeForTestMode();
|
||||
removeAllTestDictFiles();
|
||||
UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
|
||||
UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void tearDown() throws Exception {
|
||||
removeAllTestDictFiles();
|
||||
UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
|
||||
UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
|
||||
stopTestModeInNativeCode();
|
||||
|
||||
// Restore the account that was present before running the test.
|
||||
|
|
@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a random word.
|
||||
*/
|
||||
private static String generateWord(final int value) {
|
||||
final int lengthOfChars = CHARACTERS.length;
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
long lvalue = Math.abs((long)value);
|
||||
while (lvalue > 0) {
|
||||
builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
|
||||
lvalue /= lengthOfChars;
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private static List<String> generateWords(final int number, final Random random) {
|
||||
final HashSet<String> wordSet = new HashSet<>();
|
||||
while (wordSet.size() < number) {
|
||||
wordSet.add(generateWord(random.nextInt()));
|
||||
}
|
||||
return new ArrayList<>(wordSet);
|
||||
}
|
||||
|
||||
private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
|
||||
final int timestamp) {
|
||||
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
|
||||
for (final String word : words) {
|
||||
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
|
||||
DistracterFilter.EMPTY_DISTRACTER_FILTER);
|
||||
ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param checkContents if true, checks whether written words are actually in the dictionary
|
||||
* or not.
|
||||
*/
|
||||
private void addAndWriteRandomWords(final UserHistoryDictionary dict,
|
||||
final int numberOfWords, final Random random, final boolean checkContents) {
|
||||
final List<String> words = generateWords(numberOfWords, random);
|
||||
// Add random words to the user history dictionary.
|
||||
addToDict(dict, words, mCurrentTime);
|
||||
if (checkContents) {
|
||||
dict.waitAllTasksForTests();
|
||||
for (int i = 0; i < numberOfWords; ++i) {
|
||||
final String word = words.get(i);
|
||||
assertTrue(dict.isInDictionary(word));
|
||||
}
|
||||
}
|
||||
// write to file.
|
||||
dict.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all entries in the user history dictionary.
|
||||
* @param dict the user history dictionary.
|
||||
|
|
@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
public void testRandomWords() {
|
||||
Log.d(TAG, "This test can be used for profiling.");
|
||||
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
|
||||
final Locale dummyLocale = getDummyLocale("random_words");
|
||||
final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");
|
||||
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
|
||||
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
|
||||
final File dictFile = ExpandableBinaryDictionary.getDictFile(
|
||||
mContext, dictName, null /* dictFile */);
|
||||
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
|
||||
getContext(), dummyLocale, TEST_ACCOUNT);
|
||||
|
||||
clearHistory(dict);
|
||||
final int numberOfWords = 1000;
|
||||
final Random random = new Random(123456);
|
||||
clearHistory(dict);
|
||||
addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
|
||||
checkExistenceAndRemoveDictFile(dict, dictFile);
|
||||
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(
|
||||
dict, numberOfWords, random, true /* checksContents */, mCurrentTime));
|
||||
assertDictionaryExists(dict, dictFile);
|
||||
}
|
||||
|
||||
public void testStressTestForSwitchingLanguagesAndAddingWords() {
|
||||
|
|
@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
|
||||
// Create filename suffixes for this test.
|
||||
for (int i = 0; i < numberOfLanguages; i++) {
|
||||
final Locale dummyLocale = getDummyLocale("switching_languages" + i);
|
||||
final Locale dummyLocale =
|
||||
UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);
|
||||
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
|
||||
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
|
||||
dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
|
||||
|
|
@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
for (int i = 0; i < numberOfLanguageSwitching; i++) {
|
||||
final int index = i % numberOfLanguages;
|
||||
// Switch to dicts[index].
|
||||
addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
|
||||
random, false /* checksContents */);
|
||||
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],
|
||||
numberOfWordsInsertedForEachLanguageSwitch,
|
||||
random,
|
||||
false /* checksContents */,
|
||||
mCurrentTime));
|
||||
}
|
||||
|
||||
final long end = System.currentTimeMillis();
|
||||
|
|
@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
+ (end - start) + " ms");
|
||||
} finally {
|
||||
for (int i = 0; i < numberOfLanguages; i++) {
|
||||
checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
|
||||
assertDictionaryExists(dicts[i], dictFiles[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testAddManyWords() {
|
||||
final Locale dummyLocale = getDummyLocale("many_random_words");
|
||||
final Locale dummyLocale =
|
||||
UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");
|
||||
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
|
||||
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
|
||||
final File dictFile = ExpandableBinaryDictionary.getDictFile(
|
||||
|
|
@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
|
||||
getContext(), dummyLocale, TEST_ACCOUNT);
|
||||
clearHistory(dict);
|
||||
try {
|
||||
addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
|
||||
} finally {
|
||||
checkExistenceAndRemoveDictFile(dict, dictFile);
|
||||
}
|
||||
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,
|
||||
numberOfWords, random, true /* checksContents */, mCurrentTime));
|
||||
assertDictionaryExists(dict, dictFile);
|
||||
}
|
||||
|
||||
public void testDecaying() {
|
||||
final Locale dummyLocale = getDummyLocale("decaying");
|
||||
final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
|
||||
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
|
||||
getContext(), dummyLocale, TEST_ACCOUNT);
|
||||
final int numberOfWords = 5000;
|
||||
final Random random = new Random(123456);
|
||||
resetCurrentTimeForTestMode();
|
||||
clearHistory(dict);
|
||||
final List<String> words = generateWords(numberOfWords, random);
|
||||
dict.waitAllTasksForTests();
|
||||
|
||||
final int numberOfWords = 5000;
|
||||
final Random random = new Random(123456);
|
||||
final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
|
||||
random);
|
||||
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
|
||||
for (final String word : words) {
|
||||
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
|
||||
|
|
@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
|
|||
for (final String word : words) {
|
||||
assertTrue(dict.isInDictionary(word));
|
||||
}
|
||||
// Long term decay results in words removed from the dictionary.
|
||||
forcePassingLongTime();
|
||||
dict.runGCIfRequired();
|
||||
dict.waitAllTasksForTests();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,144 @@
|
|||
/*
|
||||
* Copyright (C) 2014 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.personalization;
|
||||
|
||||
import android.content.Context;
|
||||
|
||||
import com.android.inputmethod.latin.NgramContext;
|
||||
import com.android.inputmethod.latin.NgramContext.WordInfo;
|
||||
import com.android.inputmethod.latin.common.FileUtils;
|
||||
import com.android.inputmethod.latin.utils.DistracterFilter;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FilenameFilter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* Utility class for helping while running tests involving {@link UserHistoryDictionary}.
|
||||
*/
|
||||
public class UserHistoryDictionaryTestsHelper {
|
||||
|
||||
/**
|
||||
* Locale prefix for generating dummy locales for tests.
|
||||
*/
|
||||
public static final String TEST_LOCALE_PREFIX = "test-";
|
||||
|
||||
/**
|
||||
* Characters for generating random words.
|
||||
*/
|
||||
private static final String[] CHARACTERS = {
|
||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
};
|
||||
|
||||
/**
|
||||
* Remove all the test dictionary files created for the given locale.
|
||||
*/
|
||||
public static void removeAllTestDictFiles(final String filter, final Context context) {
|
||||
final FilenameFilter filenameFilter = new FilenameFilter() {
|
||||
@Override
|
||||
public boolean accept(final File dir, final String filename) {
|
||||
return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);
|
||||
}
|
||||
};
|
||||
FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates and writes random words to dictionary. Caller can be assured
|
||||
* that the write tasks would be finished; and its success would be reflected
|
||||
* in the returned boolean.
|
||||
*
|
||||
* @param dict {@link UserHistoryDictionary} to which words should be added.
|
||||
* @param numberOfWords number of words to be added.
|
||||
* @param random helps generate random words.
|
||||
* @param checkContents if true, checks whether written words are actually in the dictionary.
|
||||
* @param currentTime timestamp that would be used for adding the words.
|
||||
* @returns true if all words have been written to dictionary successfully.
|
||||
*/
|
||||
public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,
|
||||
final int numberOfWords, final Random random, final boolean checkContents,
|
||||
final int currentTime) {
|
||||
final List<String> words = generateWords(numberOfWords, random);
|
||||
// Add random words to the user history dictionary.
|
||||
addWordsToDictionary(dict, words, currentTime);
|
||||
boolean success = true;
|
||||
if (checkContents) {
|
||||
dict.waitAllTasksForTests();
|
||||
for (int i = 0; i < numberOfWords; ++i) {
|
||||
final String word = words.get(i);
|
||||
if (!dict.isInDictionary(word)) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// write to file.
|
||||
dict.close();
|
||||
dict.waitAllTasksForTests();
|
||||
return success;
|
||||
}
|
||||
|
||||
private static void addWordsToDictionary(final UserHistoryDictionary dict,
|
||||
final List<String> words, final int timestamp) {
|
||||
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
|
||||
for (final String word : words) {
|
||||
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
|
||||
DistracterFilter.EMPTY_DISTRACTER_FILTER);
|
||||
ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates unique test locale for using within tests.
|
||||
*/
|
||||
public static Locale getDummyLocale(final String name) {
|
||||
return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates random words.
|
||||
*
|
||||
* @param numberOfWords number of words to generate.
|
||||
* @param random salt used for generating random words.
|
||||
*/
|
||||
public static List<String> generateWords(final int numberOfWords, final Random random) {
|
||||
final HashSet<String> wordSet = new HashSet<>();
|
||||
while (wordSet.size() < numberOfWords) {
|
||||
wordSet.add(generateWord(random.nextInt()));
|
||||
}
|
||||
return new ArrayList<>(wordSet);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a random word.
|
||||
*/
|
||||
private static String generateWord(final int value) {
|
||||
final int lengthOfChars = CHARACTERS.length;
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
long lvalue = Math.abs((long)value);
|
||||
while (lvalue > 0) {
|
||||
builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
|
||||
lvalue /= lengthOfChars;
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue