Quit using ExpandableDictionary.

Bug: 6669677
Change-Id: Ie90417fa9b726454fe729a665fcd549efabb9e94
main
Keisuke Kuroyanagi 2013-09-24 22:57:15 +09:00
parent d1ae03046c
commit 2e58670da9
4 changed files with 181 additions and 44 deletions

View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.StringUtils; import com.android.inputmethod.latin.utils.StringUtils;
import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Locale; import java.util.Locale;
@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary {
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
} }
private void runGCIfRequired() {
if (needsToRunGCNative(mNativeDict)) {
flushWithGC();
}
}
// Add a unigram entry to binary dictionary in native code. // Add a unigram entry to binary dictionary in native code.
public void addUnigramWord(final String word, final int probability) { public void addUnigramWord(final String word, final int probability) {
if (TextUtils.isEmpty(word)) { if (TextUtils.isEmpty(word)) {
return; return;
} }
runGCIfRequired();
final int[] codePoints = StringUtils.toCodePointArray(word); final int[] codePoints = StringUtils.toCodePointArray(word);
addUnigramWordNative(mNativeDict, codePoints, probability); addUnigramWordNative(mNativeDict, codePoints, probability);
} }
@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return; return;
} }
runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1); final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return; return;
} }
runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1); final int[] codePoints1 = StringUtils.toCodePointArray(word1);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
} }
@UsedForTesting
public void flush() { public void flush() {
if (!isValidDictionary()) return; if (!isValidDictionary()) return;
flushNative(mNativeDict, mDictFilePath); flushNative(mNativeDict, mDictFilePath);
closeNative(mNativeDict);
final File dictFile = new File(mDictFilePath);
mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */);
} }
@UsedForTesting
public void flushWithGC() { public void flushWithGC() {
if (!isValidDictionary()) return; if (!isValidDictionary()) return;
flushWithGCNative(mNativeDict, mDictFilePath); flushWithGCNative(mNativeDict, mDictFilePath);
closeNative(mNativeDict);
final File dictFile = new File(mDictFilePath);
mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */);
} }
@UsedForTesting
public boolean needsToRunGC() { public boolean needsToRunGC() {
if (!isValidDictionary()) return false; if (!isValidDictionary()) return false;
return needsToRunGCNative(mNativeDict); return needsToRunGCNative(mNativeDict);

View File

@ -22,14 +22,22 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** Whether to print debug output to log */ /** Whether to print debug output to log */
private static boolean DEBUG = false; private static boolean DEBUG = false;
// TODO: Remove and enable dynamic update in native code. // TODO: Remove.
/** Whether to call binary dictionary dynamically updating methods. */ /** Whether to call binary dictionary dynamically updating methods. */
private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false; public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;
private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;
@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/ */
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
/** /**
* A static map of time recorders, each of which records the time of accesses to a single binary * A static map of time recorders, each of which records the time of accesses to a single binary
* dictionary file. The key for this map is the filename and the value is the shared dictionary * dictionary file. The key for this map is the filename and the value is the shared dictionary
@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static AbstractDictionaryWriter getDictionaryWriter(final Context context, private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
final String dictType, final boolean isDynamicPersonalizationDictionary) { final String dictType, final boolean isDynamicPersonalizationDictionary) {
if (isDynamicPersonalizationDictionary) { if (isDynamicPersonalizationDictionary) {
return new DynamicPersonalizationDictionaryWriter(context, dictType); if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
return null;
} else {
return new DynamicPersonalizationDictionaryWriter(context, dictType);
}
} else { } else {
return new DictionaryWriter(context, dictType); return new DictionaryWriter(context, dictType);
} }
@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
mBinaryDictionary.close(); mBinaryDictionary.close();
mBinaryDictionary = null; mBinaryDictionary = null;
} }
mDictionaryWriter.close(); if (mDictionaryWriter != null) {
mDictionaryWriter.close();
}
} }
}); });
} }
@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).execute(new Runnable() { getExecutor(mFilename).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
mDictionaryWriter.clear(); if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else {
mDictionaryWriter.clear();
}
} }
}); });
} }
@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() { public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addUnigramWord(word, frequency); mBinaryDictionary.addUnigramWord(word, frequency);
} else {
// TODO: Remove.
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
} }
// TODO: Remove.
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
} }
}); });
} }
@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() { public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addBigramWords(word0, word1, frequency); mBinaryDictionary.addBigramWords(word0, word1, frequency);
} else {
// TODO: Remove.
mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
0 /* lastTouchedTime */);
} }
// TODO: Remove.
mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
0 /* lastTouchedTime */);
} }
}); });
} }
@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() { public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.removeBigramWords(word0, word1); mBinaryDictionary.removeBigramWords(word0, word1);
} else {
// TODO: Remove.
mDictionaryWriter.removeBigramWords(word0, word1);
} }
// TODO: Remove.
mDictionaryWriter.removeBigramWords(word0, word1);
} }
}); });
} }
@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() { getExecutor(mFilename).executePrioritized(new Runnable() {
@Override @Override
public void run() { public void run() {
final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ? if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord, if (mBinaryDictionary == null) {
proximityInfo, blockOffensiveWords, additionalFeaturesOptions, holder.set(null);
sessionId); return;
// TODO: Remove checking mIsUpdatable and use native suggestion. }
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion = final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords, additionalFeaturesOptions, proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId); sessionId);
if (inMemDictSuggestion == null) { holder.set(binarySuggestion);
holder.set(binarySuggestion);
} else if (binarySuggestion == null) {
holder.set(inMemDictSuggestion);
} else {
binarySuggestion.addAll(inMemDictSuggestion);
holder.set(binarySuggestion);
}
} else { } else {
holder.set(inMemDictSuggestion); final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
composer.isBatchMode() ? null :
mDictionaryWriter.getSuggestionsWithSessionId(composer,
prevWord, proximityInfo, blockOffensiveWords,
additionalFeaturesOptions, sessionId);
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords,
additionalFeaturesOptions, sessionId);
if (inMemDictSuggestion == null) {
holder.set(binarySuggestion);
} else if (binarySuggestion == null) {
holder.set(inMemDictSuggestion);
} else {
binarySuggestion.addAll(inMemDictSuggestion);
holder.set(binarySuggestion);
}
} else {
holder.set(inMemDictSuggestion);
}
} }
} }
}); });
@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
true /* useFullEditDistance */, null, mDictType, mIsUpdatable); true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
// Ensure all threads accessing the current dictionary have finished before swapping in // Ensure all threads accessing the current dictionary have finished before
// the new one. // swapping in the new one.
// TODO: Ensure multi-thread assignment of mBinaryDictionary.
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
getExecutor(mFilename).executePrioritized(new Runnable() { getExecutor(mFilename).executePrioritized(new Runnable() {
@Override @Override
@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (needsToReloadBeforeWriting()) { if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear(); mDictionaryWriter.clear();
loadDictionaryAsync(); loadDictionaryAsync();
mDictionaryWriter.write(mFilename);
} else {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else {
if (mBinaryDictionary.needsToRunGC()) {
mBinaryDictionary.flushWithGC();
} else {
mBinaryDictionary.flush();
}
}
} else {
mDictionaryWriter.write(mFilename);
}
} }
mDictionaryWriter.write(mFilename);
} }
/** /**
@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() { getExecutor(mFilename).executePrioritized(new Runnable() {
@Override @Override
public void run() { public void run() {
loadDictionaryAsync(); if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
loadDictionaryAsync();
}
} }
}); });
} }
@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** /**
* Generate binary dictionary using DictionaryWriter. * Generate binary dictionary using DictionaryWriter.
*/ */
protected void asyncWriteBinaryDictionary() { protected void asyncFlashAllBinaryDictionary() {
final Runnable newTask = new Runnable() { final Runnable newTask = new Runnable() {
@Override @Override
public void run() { public void run() {
@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@Override @Override
public void run() { public void run() {
if (mDictType == Dictionary.TYPE_USER_HISTORY) { if (mDictType == Dictionary.TYPE_USER_HISTORY) {
holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
.isInDictionaryForTests(word)); holder.set(mBinaryDictionary.isValidWord(word));
} else {
holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
.isInDictionaryForTests(word));
}
} }
} }
}); });

View File

@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
@Override @Override
public void close() { public void close() {
// Close only binary dictionary to reuse this dictionary. if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
// super.close(); closeBinaryDictionary();
closeBinaryDictionary(); }
// Flush pending writes. // Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary. // TODO: Remove after this class become to use a dynamic binary dictionary.
asyncWriteBinaryDictionary(); asyncFlashAllBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
} }
@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Clear the node structure on memory // Clear the node structure on memory
clear(); clear();
// Then flush the cleared state of the dictionary on disk. // Then flush the cleared state of the dictionary on disk.
asyncWriteBinaryDictionary(); asyncFlashAllBinaryDictionary();
} }
} }

View File

@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) { } catch (InterruptedException e) {
} }
for (int i = 0; i < 10 && i < numberOfWords; ++i) { // Limit word count to check when using a Java on memory dictionary.
final int wordCountToCheck =
ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
numberOfWords : 10;
for (int i = 0; i < wordCountToCheck; ++i) {
final String word = words.get(i); final String word = words.get(i);
// This may fail as long as we use tryLock on inserting the bigram words // This may fail as long as we use tryLock on inserting the bigram words
assertTrue(dict.isInDictionaryForTests(word)); assertTrue(dict.isInDictionaryForTests(word));
@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
} }
} }
} }
public void testAddManyWords() {
File dictFile = null;
final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
final int numberOfWords =
ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
10000 : 1000;
final Random random = new Random(123456);
UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
testFilenameSuffix, mPrefs);
try {
addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
true /* checksContents */);
dict.close();
} finally {
try {
Log.d(TAG, "waiting for writing ...");
dict.shutdownExecutorForTests();
while (!dict.isTerminatedForTests()) {
Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
}
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: ", e);
}
final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {
assertTrue(dictFile.exists());
assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
dictFile.delete();
}
}
}
} }