From 2e58670da9687fd1fd28c322e03343957d11568c Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Tue, 24 Sep 2013 22:57:15 +0900 Subject: [PATCH] Quit using ExpandableDictionary. Bug: 6669677 Change-Id: Ie90417fa9b726454fe729a665fcd549efabb9e94 --- .../inputmethod/latin/BinaryDictionary.java | 21 ++- .../latin/ExpandableBinaryDictionary.java | 151 ++++++++++++++---- .../DynamicPredictionDictionaryBase.java | 10 +- .../UserHistoryDictionaryTests.java | 43 ++++- 4 files changed, 181 insertions(+), 44 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 632ee0da4..61ccfcfad 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.StringUtils; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; @@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary { return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); } + private void runGCIfRequired() { + if (needsToRunGCNative(mNativeDict)) { + flushWithGC(); + } + } + // Add a unigram entry to binary dictionary in native code. public void addUnigramWord(final String word, final int probability) { if (TextUtils.isEmpty(word)) { return; } + runGCIfRequired(); final int[] codePoints = StringUtils.toCodePointArray(word); addUnigramWordNative(mNativeDict, codePoints, probability); } @@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } + runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); @@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } + runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); } - @UsedForTesting public void flush() { if (!isValidDictionary()) return; flushNative(mNativeDict, mDictFilePath); + closeNative(mNativeDict); + final File dictFile = new File(mDictFilePath); + mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, + dictFile.length(), true /* isUpdatable */); } - @UsedForTesting public void flushWithGC() { if (!isValidDictionary()) return; flushWithGCNative(mNativeDict, mDictFilePath); + closeNative(mNativeDict); + final File dictFile = new File(mDictFilePath); + mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, + dictFile.length(), true /* isUpdatable */); } - @UsedForTesting public boolean needsToRunGC() { if (!isValidDictionary()) return false; return needsToRunGCNative(mNativeDict); diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index fcd7ede1a..0774ce203 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,14 +22,22 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; +import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; - // TODO: Remove and enable dynamic update in native code. + // TODO: Remove. /** Whether to call binary dictionary dynamically updating methods. */ - private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false; + public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true; private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; @@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); + /** * A static map of time recorders, each of which records the time of accesses to a single binary * dictionary file. The key for this map is the filename and the value is the shared dictionary @@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static AbstractDictionaryWriter getDictionaryWriter(final Context context, final String dictType, final boolean isDynamicPersonalizationDictionary) { if (isDynamicPersonalizationDictionary) { - return new DynamicPersonalizationDictionaryWriter(context, dictType); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + return null; + } else { + return new DynamicPersonalizationDictionaryWriter(context, dictType); + } } else { return new DictionaryWriter(context, dictType); } @@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary.close(); mBinaryDictionary = null; } - mDictionaryWriter.close(); + if (mDictionaryWriter != null) { + mDictionaryWriter.close(); + } } }); } @@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - mDictionaryWriter.clear(); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { + mBinaryDictionary.close(); + final File file = new File(mContext.getFilesDir(), mFilename); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + new FusionDictionary.DictionaryOptions(new HashMap(), + false, false)); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + try { + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + } catch (IOException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } catch (UnsupportedFormatException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } + } else { + mDictionaryWriter.clear(); + } } }); } @@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.addUnigramWord(word, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); } - // TODO: Remove. - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); } }); } @@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.addBigramWords(word0, word1, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, + 0 /* lastTouchedTime */); } - // TODO: Remove. - mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, - 0 /* lastTouchedTime */); } }); } @@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.removeBigramWords(word0, word1); + } else { + // TODO: Remove. + mDictionaryWriter.removeBigramWords(word0, word1); } - // TODO: Remove. - mDictionaryWriter.removeBigramWords(word0, word1); } }); } @@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - final ArrayList inMemDictSuggestion = composer.isBatchMode() ? - null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, - sessionId); - // TODO: Remove checking mIsUpdatable and use native suggestion. - if (mBinaryDictionary != null && !mIsUpdatable) { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null) { + holder.set(null); + return; + } final ArrayList binarySuggestion = mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId); - if (inMemDictSuggestion == null) { - holder.set(binarySuggestion); - } else if (binarySuggestion == null) { - holder.set(inMemDictSuggestion); - } else { - binarySuggestion.addAll(inMemDictSuggestion); - holder.set(binarySuggestion); - } + holder.set(binarySuggestion); } else { - holder.set(inMemDictSuggestion); + final ArrayList inMemDictSuggestion = + composer.isBatchMode() ? null : + mDictionaryWriter.getSuggestionsWithSessionId(composer, + prevWord, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + // TODO: Remove checking mIsUpdatable and use native suggestion. + if (mBinaryDictionary != null && !mIsUpdatable) { + final ArrayList binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + if (inMemDictSuggestion == null) { + holder.set(binarySuggestion); + } else if (binarySuggestion == null) { + holder.set(inMemDictSuggestion); + } else { + binarySuggestion.addAll(inMemDictSuggestion); + holder.set(binarySuggestion); + } + } else { + holder.set(inMemDictSuggestion); + } } } }); @@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - // Ensure all threads accessing the current dictionary have finished before swapping in - // the new one. + // Ensure all threads accessing the current dictionary have finished before + // swapping in the new one. + // TODO: Ensure multi-thread assignment of mBinaryDictionary. final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; getExecutor(mFilename).executePrioritized(new Runnable() { @Override @@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); loadDictionaryAsync(); + mDictionaryWriter.write(mFilename); + } else { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { + final File file = new File(mContext.getFilesDir(), mFilename); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + new FusionDictionary.DictionaryOptions(new HashMap(), + false, false)); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + try { + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + } catch (IOException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } catch (UnsupportedFormatException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } + } else { + if (mBinaryDictionary.needsToRunGC()) { + mBinaryDictionary.flushWithGC(); + } else { + mBinaryDictionary.flush(); + } + } + } else { + mDictionaryWriter.write(mFilename); + } } - mDictionaryWriter.write(mFilename); } /** @@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - loadDictionaryAsync(); + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + loadDictionaryAsync(); + } } }); } @@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Generate binary dictionary using DictionaryWriter. */ - protected void asyncWriteBinaryDictionary() { + protected void asyncFlashAllBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { @@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public void run() { if (mDictType == Dictionary.TYPE_USER_HISTORY) { - holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInDictionaryForTests(word)); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + holder.set(mBinaryDictionary.isValidWord(word)); + } else { + holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) + .isInDictionaryForTests(word)); + } } } }); diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 9364fb034..075d7e3c3 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi @Override public void close() { - // Close only binary dictionary to reuse this dictionary. - // super.close(); - closeBinaryDictionary(); + if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + closeBinaryDictionary(); + } // Flush pending writes. // TODO: Remove after this class become to use a dynamic binary dictionary. - asyncWriteBinaryDictionary(); + asyncFlashAllBinaryDictionary(); Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); } @@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Clear the node structure on memory clear(); // Then flush the cleared state of the dictionary on disk. - asyncWriteBinaryDictionary(); + asyncFlashAllBinaryDictionary(); } } diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index bf44a1424..d605cdb84 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { } - for (int i = 0; i < 10 && i < numberOfWords; ++i) { + // Limit word count to check when using a Java on memory dictionary. + final int wordCountToCheck = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + numberOfWords : 10; + for (int i = 0; i < wordCountToCheck; ++i) { final String word = words.get(i); // This may fail as long as we use tryLock on inserting the bigram words assertTrue(dict.isInDictionaryForTests(word)); @@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } } } + + public void testAddManyWords() { + File dictFile = null; + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + 10000 : 1000; + final Random random = new Random(123456); + + UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + true /* checksContents */); + dict.close(); + } finally { + try { + Log.d(TAG, "waiting for writing ..."); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } catch (InterruptedException e) { + Log.d(TAG, "InterruptedException: ", e); + } + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); + if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); + dictFile.delete(); + } + } + } + }