nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 2e58670d: Quit using ExpandableDictionary.

Browse source 
* commit '2e58670da9687fd1fd28c322e03343957d11568c':
  Quit using ExpandableDictionary.
This commit is contained in:
Keisuke Kuroyanagi 2013-09-24 07:08:08 -07:00 committed by Android Git Automerger
commit f05aeb64d9
4 changed files with 181 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.StringUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Locale;
@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary {
return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
}
private void runGCIfRequired() {
if (needsToRunGCNative(mNativeDict)) {
flushWithGC();
}
}
// Add a unigram entry to binary dictionary in native code.
public void addUnigramWord(final String word, final int probability) {
if (TextUtils.isEmpty(word)) {
return;
}
runGCIfRequired();
final int[] codePoints = StringUtils.toCodePointArray(word);
addUnigramWordNative(mNativeDict, codePoints, probability);
}
@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary {
if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
return;
}
runGCIfRequired();
final int[] codePoints0 = StringUtils.toCodePointArray(word0);
final int[] codePoints1 = StringUtils.toCodePointArray(word1);
removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
}
@UsedForTesting
public void flush() {
if (!isValidDictionary()) return;
flushNative(mNativeDict, mDictFilePath);
closeNative(mNativeDict);
final File dictFile = new File(mDictFilePath);
mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */);
}
@UsedForTesting
public void flushWithGC() {
if (!isValidDictionary()) return;
flushWithGCNative(mNativeDict, mDictFilePath);
closeNative(mNativeDict);
final File dictFile = new File(mDictFilePath);
mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */);
}
@UsedForTesting
public boolean needsToRunGC() {
if (!isValidDictionary()) return false;
return needsToRunGCNative(mNativeDict);

151
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View file

@ -22,14 +22,22 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;
@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** Whether to print debug output to log */
private static boolean DEBUG = false;
// TODO: Remove and enable dynamic update in native code.
// TODO: Remove.
/** Whether to call binary dictionary dynamically updating methods. */
private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false;
public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;
private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;
@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
/**
* A static map of time recorders, each of which records the time of accesses to a single binary
* dictionary file. The key for this map is the filename and the value is the shared dictionary
@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
final String dictType, final boolean isDynamicPersonalizationDictionary) {
if (isDynamicPersonalizationDictionary) {
return new DynamicPersonalizationDictionaryWriter(context, dictType);
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
return null;
} else {
return new DynamicPersonalizationDictionaryWriter(context, dictType);
}
} else {
return new DictionaryWriter(context, dictType);
}
@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
mBinaryDictionary.close();
mBinaryDictionary = null;
}
mDictionaryWriter.close();
if (mDictionaryWriter != null) {
mDictionaryWriter.close();
}
}
});
}
@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).execute(new Runnable() {
@Override
public void run() {
mDictionaryWriter.clear();
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else {
mDictionaryWriter.clear();
}
}
});
}
@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addUnigramWord(word, frequency);
} else {
// TODO: Remove.
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
// TODO: Remove.
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
});
}
@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.addBigramWords(word0, word1, frequency);
} else {
// TODO: Remove.
mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
0 /* lastTouchedTime */);
}
// TODO: Remove.
mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
0 /* lastTouchedTime */);
}
});
}
@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
mBinaryDictionary.removeBigramWords(word0, word1);
} else {
// TODO: Remove.
mDictionaryWriter.removeBigramWords(word0, word1);
}
// TODO: Remove.
mDictionaryWriter.removeBigramWords(word0, word1);
}
});
}
@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId);
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null) {
holder.set(null);
return;
}
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId);
if (inMemDictSuggestion == null) {
holder.set(binarySuggestion);
} else if (binarySuggestion == null) {
holder.set(inMemDictSuggestion);
} else {
binarySuggestion.addAll(inMemDictSuggestion);
holder.set(binarySuggestion);
}
holder.set(binarySuggestion);
} else {
holder.set(inMemDictSuggestion);
final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
composer.isBatchMode() ? null :
mDictionaryWriter.getSuggestionsWithSessionId(composer,
prevWord, proximityInfo, blockOffensiveWords,
additionalFeaturesOptions, sessionId);
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
proximityInfo, blockOffensiveWords,
additionalFeaturesOptions, sessionId);
if (inMemDictSuggestion == null) {
holder.set(binarySuggestion);
} else if (binarySuggestion == null) {
holder.set(inMemDictSuggestion);
} else {
binarySuggestion.addAll(inMemDictSuggestion);
holder.set(binarySuggestion);
}
} else {
holder.set(inMemDictSuggestion);
}
}
}
});
@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
// Ensure all threads accessing the current dictionary have finished before swapping in
// the new one.
// Ensure all threads accessing the current dictionary have finished before
// swapping in the new one.
// TODO: Ensure multi-thread assignment of mBinaryDictionary.
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
mDictionaryWriter.write(mFilename);
} else {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else {
if (mBinaryDictionary.needsToRunGC()) {
mBinaryDictionary.flushWithGC();
} else {
mBinaryDictionary.flush();
}
}
} else {
mDictionaryWriter.write(mFilename);
}
}
mDictionaryWriter.write(mFilename);
}
/**
@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mFilename).executePrioritized(new Runnable() {
@Override
public void run() {
loadDictionaryAsync();
if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
loadDictionaryAsync();
}
}
});
}
@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Generate binary dictionary using DictionaryWriter.
*/
protected void asyncWriteBinaryDictionary() {
protected void asyncFlashAllBinaryDictionary() {
final Runnable newTask = new Runnable() {
@Override
public void run() {
@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@Override
public void run() {
if (mDictType == Dictionary.TYPE_USER_HISTORY) {
holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
.isInDictionaryForTests(word));
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
holder.set(mBinaryDictionary.isValidWord(word));
} else {
holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
.isInDictionaryForTests(word));
}
}
}
});

10
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View file

@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
@Override
public void close() {
// Close only binary dictionary to reuse this dictionary.
// super.close();
closeBinaryDictionary();
if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
closeBinaryDictionary();
}
// Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary.
asyncWriteBinaryDictionary();
asyncFlashAllBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
}
@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Clear the node structure on memory
clear();
// Then flush the cleared state of the dictionary on disk.
asyncWriteBinaryDictionary();
asyncFlashAllBinaryDictionary();
}
}

43
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View file

@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
} catch (InterruptedException e) {
}
for (int i = 0; i < 10 && i < numberOfWords; ++i) {
// Limit word count to check when using a Java on memory dictionary.
final int wordCountToCheck =
ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
numberOfWords : 10;
for (int i = 0; i < wordCountToCheck; ++i) {
final String word = words.get(i);
// This may fail as long as we use tryLock on inserting the bigram words
assertTrue(dict.isInDictionaryForTests(word));
@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
}
}
public void testAddManyWords() {
File dictFile = null;
final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
final int numberOfWords =
ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
10000 : 1000;
final Random random = new Random(123456);
UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
testFilenameSuffix, mPrefs);
try {
addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
true /* checksContents */);
dict.close();
} finally {
try {
Log.d(TAG, "waiting for writing ...");
dict.shutdownExecutorForTests();
while (!dict.isTerminatedForTests()) {
Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
}
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: ", e);
}
final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {
assertTrue(dictFile.exists());
assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
dictFile.delete();
}
}
}
}