nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Hook for fetching sync content from UserHistoryDict"

main
Jatin Matani 2014-12-08 20:00:09 +00:00 committed by Android (Google) Code Review
parent 639306faf1 12d63820d4
commit 2bdd5290a9
4 changed files with 213 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;
private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =
new WordProperty[0] /* default */;
/** The application context. */
protected final Context mContext;
@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
});
}
/**
* Returns dictionary content required for syncing.
*/
public WordProperty[] getWordPropertiesForSyncing() {
reloadDictionaryIfRequired();
final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();
asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {
@Override
public void run() {
final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();
final BinaryDictionary binaryDictionary = getBinaryDictionary();
if (binaryDictionary == null) {
return;
}
int token = 0;
do {
// TODO: We need a new API that returns *new* un-synced data.
final BinaryDictionary.GetNextWordPropertyResult result =
binaryDictionary.getNextWordProperty(token);
final WordProperty wordProperty = result.mWordProperty;
if (wordProperty == null) {
break;
}
wordPropertyList.add(wordProperty);
token = result.mNextToken;
} while (token != 0);
result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));
}
});
// TODO: Figure out the best timeout duration for this API.
return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,
TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
}
}

3
java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
View File

@ -58,8 +58,7 @@ public class PersonalizationHelper {
final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (DEBUG) {
Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +
" & account" + accountName);
Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);
}
dict.reloadDictionaryIfRequired();
return dict;

136
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -25,15 +25,11 @@ import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.NgramContext;
import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.common.FileUtils;
import com.android.inputmethod.latin.settings.LocalSettingsConstants;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@ -48,34 +44,13 @@ import javax.annotation.Nullable;
public class UserHistoryDictionaryTests extends AndroidTestCase {
private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
private static final String TEST_LOCALE_PREFIX = "test_";
private static final String TEST_ACCOUNT = "account@example.com";
private static final String[] CHARACTERS = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
};
private int mCurrentTime = 0;
private SharedPreferences mPrefs;
private String mLastKnownAccount = null;
private void removeAllTestDictFiles() {
final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
final File dictFile = ExpandableBinaryDictionary.getDictFile(
mContext, dictName, null /* dictFile */);
final FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(final File dir, final String filename) {
return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
}
};
FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
}
private static void printAllFiles(final File dir) {
Log.d(TAG, dir.getAbsolutePath());
for (final File file : dir.listFiles()) {
@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
}
private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
private static void assertDictionaryExists(final UserHistoryDictionary dict,
final File dictFile) {
Log.d(TAG, "waiting for writing ...");
dict.waitAllTasksForTests();
@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Log.e(TAG, "Interrupted during waiting for writing the dict file.");
}
}
assertTrue("check exisiting of " + dictFile, dictFile.exists());
FileUtils.deleteRecursively(dictFile);
}
private static Locale getDummyLocale(final String name) {
return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());
}
@Override
@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
updateAccountName(TEST_ACCOUNT);
resetCurrentTimeForTestMode();
removeAllTestDictFiles();
UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
}
@Override
protected void tearDown() throws Exception {
removeAllTestDictFiles();
UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
stopTestModeInNativeCode();
// Restore the account that was present before running the test.
@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
/**
* Generates a random word.
*/
private static String generateWord(final int value) {
final int lengthOfChars = CHARACTERS.length;
final StringBuilder builder = new StringBuilder();
long lvalue = Math.abs((long)value);
while (lvalue > 0) {
builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
lvalue /= lengthOfChars;
}
return builder.toString();
}
private static List<String> generateWords(final int number, final Random random) {
final HashSet<String> wordSet = new HashSet<>();
while (wordSet.size() < number) {
wordSet.add(generateWord(random.nextInt()));
}
return new ArrayList<>(wordSet);
}
private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
final int timestamp) {
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
}
}
/**
* @param checkContents if true, checks whether written words are actually in the dictionary
* or not.
*/
private void addAndWriteRandomWords(final UserHistoryDictionary dict,
final int numberOfWords, final Random random, final boolean checkContents) {
final List<String> words = generateWords(numberOfWords, random);
// Add random words to the user history dictionary.
addToDict(dict, words, mCurrentTime);
if (checkContents) {
dict.waitAllTasksForTests();
for (int i = 0; i < numberOfWords; ++i) {
final String word = words.get(i);
assertTrue(dict.isInDictionary(word));
}
}
// write to file.
dict.close();
}
/**
* Clear all entries in the user history dictionary.
* @param dict the user history dictionary.
@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
public void testRandomWords() {
Log.d(TAG, "This test can be used for profiling.");
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
final Locale dummyLocale = getDummyLocale("random_words");
final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
final File dictFile = ExpandableBinaryDictionary.getDictFile(
mContext, dictName, null /* dictFile */);
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
clearHistory(dict);
final int numberOfWords = 1000;
final Random random = new Random(123456);
clearHistory(dict);
addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
checkExistenceAndRemoveDictFile(dict, dictFile);
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(
dict, numberOfWords, random, true /* checksContents */, mCurrentTime));
assertDictionaryExists(dict, dictFile);
}
public void testStressTestForSwitchingLanguagesAndAddingWords() {
@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
final Locale dummyLocale = getDummyLocale("switching_languages" + i);
final Locale dummyLocale =
UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
for (int i = 0; i < numberOfLanguageSwitching; i++) {
final int index = i % numberOfLanguages;
// Switch to dicts[index].
addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
random, false /* checksContents */);
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],
numberOfWordsInsertedForEachLanguageSwitch,
random,
false /* checksContents */,
mCurrentTime));
}
final long end = System.currentTimeMillis();
@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
+ (end - start) + " ms");
} finally {
for (int i = 0; i < numberOfLanguages; i++) {
checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
assertDictionaryExists(dicts[i], dictFiles[i]);
}
}
}
public void testAddManyWords() {
final Locale dummyLocale = getDummyLocale("many_random_words");
final Locale dummyLocale =
UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");
final String dictName = UserHistoryDictionary.getUserHistoryDictName(
UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
final File dictFile = ExpandableBinaryDictionary.getDictFile(
@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
clearHistory(dict);
try {
addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
} finally {
checkExistenceAndRemoveDictFile(dict, dictFile);
}
assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,
numberOfWords, random, true /* checksContents */, mCurrentTime));
assertDictionaryExists(dict, dictFile);
}
public void testDecaying() {
final Locale dummyLocale = getDummyLocale("decaying");
final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
getContext(), dummyLocale, TEST_ACCOUNT);
final int numberOfWords = 5000;
final Random random = new Random(123456);
resetCurrentTimeForTestMode();
clearHistory(dict);
final List<String> words = generateWords(numberOfWords, random);
dict.waitAllTasksForTests();
final int numberOfWords = 5000;
final Random random = new Random(123456);
final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
random);
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
for (final String word : words) {
assertTrue(dict.isInDictionary(word));
}
// Long term decay results in words removed from the dictionary.
forcePassingLongTime();
dict.runGCIfRequired();
dict.waitAllTasksForTests();

144
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java Normal file
View File

@ -0,0 +1,144 @@
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.personalization;
import android.content.Context;
import com.android.inputmethod.latin.NgramContext;
import com.android.inputmethod.latin.NgramContext.WordInfo;
import com.android.inputmethod.latin.common.FileUtils;
import com.android.inputmethod.latin.utils.DistracterFilter;
import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Random;
/**
* Utility class for helping while running tests involving {@link UserHistoryDictionary}.
*/
public class UserHistoryDictionaryTestsHelper {
/**
* Locale prefix for generating dummy locales for tests.
*/
public static final String TEST_LOCALE_PREFIX = "test-";
/**
* Characters for generating random words.
*/
private static final String[] CHARACTERS = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
};
/**
* Remove all the test dictionary files created for the given locale.
*/
public static void removeAllTestDictFiles(final String filter, final Context context) {
final FilenameFilter filenameFilter = new FilenameFilter() {
@Override
public boolean accept(final File dir, final String filename) {
return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);
}
};
FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);
}
/**
* Generates and writes random words to dictionary. Caller can be assured
* that the write tasks would be finished; and its success would be reflected
* in the returned boolean.
*
* @param dict {@link UserHistoryDictionary} to which words should be added.
* @param numberOfWords number of words to be added.
* @param random helps generate random words.
* @param checkContents if true, checks whether written words are actually in the dictionary.
* @param currentTime timestamp that would be used for adding the words.
* @returns true if all words have been written to dictionary successfully.
*/
public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,
final int numberOfWords, final Random random, final boolean checkContents,
final int currentTime) {
final List<String> words = generateWords(numberOfWords, random);
// Add random words to the user history dictionary.
addWordsToDictionary(dict, words, currentTime);
boolean success = true;
if (checkContents) {
dict.waitAllTasksForTests();
for (int i = 0; i < numberOfWords; ++i) {
final String word = words.get(i);
if (!dict.isInDictionary(word)) {
success = false;
break;
}
}
}
// write to file.
dict.close();
dict.waitAllTasksForTests();
return success;
}
private static void addWordsToDictionary(final UserHistoryDictionary dict,
final List<String> words, final int timestamp) {
NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
for (final String word : words) {
UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
DistracterFilter.EMPTY_DISTRACTER_FILTER);
ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
}
}
/**
* Creates unique test locale for using within tests.
*/
public static Locale getDummyLocale(final String name) {
return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
}
/**
* Generates random words.
*
* @param numberOfWords number of words to generate.
* @param random salt used for generating random words.
*/
public static List<String> generateWords(final int numberOfWords, final Random random) {
final HashSet<String> wordSet = new HashSet<>();
while (wordSet.size() < numberOfWords) {
wordSet.add(generateWord(random.nextInt()));
}
return new ArrayList<>(wordSet);
}
/**
* Generates a random word.
*/
private static String generateWord(final int value) {
final int lengthOfChars = CHARACTERS.length;
final StringBuilder builder = new StringBuilder();
long lvalue = Math.abs((long)value);
while (lvalue > 0) {
builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
lvalue /= lengthOfChars;
}
return builder.toString();
}
}