From ea89cb40b7641764d385bbc200c5910e08a29c1a Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Thu, 27 Feb 2014 23:21:09 +0900 Subject: [PATCH] Use dynamic operations to construct all ver4 dicts. Bug: 8187060 Bug: 13127350 Change-Id: I081ee904c41898128efa8ba7a1bf3fa0a46c6231 --- .../latin/AbstractDictionaryWriter.java | 72 ----- .../inputmethod/latin/BinaryDictionary.java | 1 - .../latin/ContactsBinaryDictionary.java | 56 ++-- .../inputmethod/latin/DictionaryWriter.java | 92 ------ .../latin/ExpandableBinaryDictionary.java | 290 ++++++------------ .../latin/UserBinaryDictionary.java | 29 +- .../latin/inputlogic/InputLogicHandler.java | 2 +- .../latin/makedict/DictEncoder.java | 2 + ...ecayingExpandableBinaryDictionaryBase.java | 20 +- 9 files changed, 150 insertions(+), 414 deletions(-) delete mode 100644 java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java delete mode 100644 java/src/com/android/inputmethod/latin/DictionaryWriter.java diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java deleted file mode 100644 index 1c6a14efe..000000000 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import android.util.Log; - -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver4DictEncoder; -import com.android.inputmethod.latin.utils.FileUtils; - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -abstract public class AbstractDictionaryWriter { - /** Used for Log actions from this class */ - private static final String TAG = AbstractDictionaryWriter.class.getSimpleName(); - - public AbstractDictionaryWriter() { - } - - abstract public void clear(); - - /** - * Add a unigram with an optional shortcut to the dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - abstract public void addUnigramWord(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord); - - // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve. - abstract public void addBigramWords(final String word0, final String word1, - final int frequency, final boolean isValid, final long lastModifiedTime); - - abstract public void removeBigramWords(final String word0, final String word1); - - abstract protected void writeDictionary(final DictEncoder dictEncoder, - final Map attributeMap) throws IOException, UnsupportedFormatException; - - public void write(final File file, final Map attributeMap) { - try { - FileUtils.deleteRecursively(file); - file.mkdir(); - final DictEncoder dictEncoder = new Ver4DictEncoder(file); - writeDictionary(dictEncoder, attributeMap); - } catch (IOException e) { - Log.e(TAG, "IO exception while writing file", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Unsupported format", e); - } - } -} diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index c450a1d4f..8cdddeb10 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -30,7 +30,6 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.StringUtils; diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index ae9bdf3fc..c2941e424 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -78,7 +78,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { public ContactsBinaryDictionary(final Context context, final Locale locale, final File dictFile) { super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS, - false /* isUpdatable */, dictFile); + dictFile); mLocale = locale; mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); registerObserver(context); @@ -114,14 +114,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } @Override - public void loadDictionaryAsync() { - loadDeviceAccountsEmailAddresses(); - loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI); + public void loadInitialContentsLocked() { + loadDeviceAccountsEmailAddressesLocked(); + loadDictionaryForUriLocked(ContactsContract.Profile.CONTENT_URI); // TODO: Switch this URL to the newer ContactsContract too - loadDictionaryAsyncForUri(Contacts.CONTENT_URI); + loadDictionaryForUriLocked(Contacts.CONTENT_URI); } - private void loadDeviceAccountsEmailAddresses() { + private void loadDeviceAccountsEmailAddressesLocked() { final List accountVocabulary = AccountUtils.getDeviceAccountsEmailAddresses(mContext); if (accountVocabulary == null || accountVocabulary.isEmpty()) { @@ -131,12 +131,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (DEBUG) { Log.d(TAG, "loadAccountVocabulary: " + word); } - super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, 0 /* shortcutFreq */, - false /* isNotAWord */); + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, + 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); } } - private void loadDictionaryAsyncForUri(final Uri uri) { + private void loadDictionaryForUriLocked(final Uri uri) { Cursor cursor = null; try { cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null); @@ -145,7 +147,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } if (cursor.moveToFirst()) { sContactCountAtLastRebuild = getContactCount(); - addWords(cursor); + addWordsLocked(cursor); } } catch (final SQLiteException e) { Log.e(TAG, "SQLiteException in the remote Contacts process.", e); @@ -166,12 +168,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { return false; } - private void addWords(final Cursor cursor) { + private void addWordsLocked(final Cursor cursor) { int count = 0; while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) { String name = cursor.getString(INDEX_NAME); if (isValidName(name)) { - addName(name); + addNameLocked(name); ++count; } else { if (DEBUG_DUMP) { @@ -207,7 +209,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { * Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their * bigrams depending on locale. */ - private void addName(final String name) { + private void addNameLocked(final String name) { int len = StringUtils.codePointCount(name); String prevWord = null; // TODO: Better tokenization for non-Latin writing systems @@ -226,13 +228,15 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (DEBUG) { Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord); } - super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, - 0 /* shortcutFreq */, false /* isNotAWord */); - if (!TextUtils.isEmpty(prevWord)) { - if (mUseFirstLastBigrams) { - super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM, - 0 /* lastModifiedTime */); - } + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, + null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addBigramDynamicallyLocked(prevWord, word, + FREQUENCY_FOR_CONTACTS_BIGRAM, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); } prevWord = word; } @@ -258,12 +262,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return true; } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { final long startTime = SystemClock.uptimeMillis(); final int contactCount = getContactCount(); if (contactCount > MAX_CONTACT_COUNT) { @@ -291,7 +295,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (cursor.moveToFirst()) { while (!cursor.isAfterLast()) { String name = cursor.getString(INDEX_NAME); - if (isValidName(name) && !isNameInDictionary(name)) { + if (isValidName(name) && !isNameInDictionaryLocked(name)) { if (DEBUG) { Log.d(TAG, "Contact name missing: " + name + " (runtime = " + (SystemClock.uptimeMillis() - startTime) + " ms)"); @@ -321,7 +325,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { /** * Checks if the words in a name are in the current binary dictionary. */ - private boolean isNameInDictionary(final String name) { + private boolean isNameInDictionaryLocked(final String name) { int len = StringUtils.codePointCount(name); String prevWord = null; for (int i = 0; i < len; i++) { @@ -332,11 +336,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { final int wordLen = StringUtils.codePointCount(word); if (wordLen < MAX_WORD_LENGTH && wordLen > 1) { if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { - if (!super.isValidBigramLocked(prevWord, word)) { + if (!isValidBigramLocked(prevWord, word)) { return false; } } else { - if (!super.isValidWordLocked(word)) { + if (!isValidWordLocked(word)) { return false; } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java deleted file mode 100644 index b931c66d1..000000000 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.makedict.ProbabilityInfo; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -/** - * An in memory dictionary for memorizing entries and writing a binary dictionary. - */ -public class DictionaryWriter extends AbstractDictionaryWriter { - private static final int BINARY_DICT_VERSION = FormatSpec.VERSION4; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION, false /* hasTimestamp */); - - private FusionDictionary mFusionDictionary; - - public DictionaryWriter() { - clear(); - } - - @Override - public void clear() { - final HashMap attributes = CollectionUtils.newHashMap(); - mFusionDictionary = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(attributes)); - } - - /** - * Adds a word unigram to the fusion dictionary. - */ - // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries, - // considering performance regression. - @Override - public void addUnigramWord(final String word, final String shortcutTarget, - final int probability, final int shortcutProbability, final boolean isNotAWord) { - if (shortcutTarget == null) { - mFusionDictionary.add(word, new ProbabilityInfo(probability), null, isNotAWord); - } else { - // TODO: Do this in the subclass, with this class taking an arraylist. - final ArrayList shortcutTargets = CollectionUtils.newArrayList(); - shortcutTargets.add(new WeightedString(shortcutTarget, shortcutProbability)); - mFusionDictionary.add(word, new ProbabilityInfo(probability), shortcutTargets, - isNotAWord); - } - } - - @Override - public void addBigramWords(final String word0, final String word1, final int probability, - final boolean isValid, final long lastModifiedTime) { - mFusionDictionary.setBigram(word0, word1, new ProbabilityInfo(probability)); - } - - @Override - public void removeBigramWords(final String word0, final String word1) { - // This class don't support removing bigram words. - } - - @Override - protected void writeDictionary(final DictEncoder dictEncoder, - final Map attributeMap) throws IOException, UnsupportedFormatException { - for (final Map.Entry entry : attributeMap.entrySet()) { - mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue()); - } - dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); - } -} diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index c3e33c0c1..b18951500 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -32,7 +32,6 @@ import com.android.inputmethod.latin.utils.CombinedFormatUtils; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; -import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; import java.util.ArrayList; @@ -90,10 +89,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private BinaryDictionary mBinaryDictionary; - // TODO: Remove and handle dictionaries in native code. - /** The in-memory dictionary used to generate the binary dictionary. */ - protected AbstractDictionaryWriter mDictionaryWriter; - /** * The name of this dictionary, used as a part of the filename for storing the binary * dictionary. Multiple dictionary instances with the same name is supported, with access @@ -104,9 +99,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Dictionary locale */ private final Locale mLocale; - /** Whether to support dynamically updating the dictionary */ - private final boolean mIsUpdatable; - /** Dictionary file */ private final File mDictFile; @@ -126,23 +118,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { new AtomicReference(); /** - * Abstract method for loading the unigrams and bigrams of a given dictionary in a background - * thread. + * Abstract method for loading initial contents of a given dictionary. */ - protected abstract void loadDictionaryAsync(); + protected abstract void loadInitialContentsLocked(); /** - * Indicates that the source dictionary content has changed and a rebuild of the binary file is - * required. If it returns false, the next reload will only read the current binary dictionary - * from file. Note that the shared binary dictionary is locked when this is called. + * Indicates that the source dictionary contents have changed and a rebuild of the binary file + * is required. If it returns false, the next reload will only read the current binary + * dictionary from file. Note that the shared binary dictionary is locked when this is called. */ - protected abstract boolean hasContentChanged(); + protected abstract boolean haveContentsChanged(); private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { return formatVersion == FormatSpec.VERSION4; } - public boolean isValidDictionary() { + public boolean isValidDictionaryLocked() { return mBinaryDictionary.isValidDictionary(); } @@ -161,15 +152,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return recorder; } - private static AbstractDictionaryWriter getDictionaryWriter( - final boolean isDynamicPersonalizationDictionary) { - if (isDynamicPersonalizationDictionary) { - return null; - } else { - return new DictionaryWriter(); - } - } - /** * Creates a new expandable binary dictionary. * @@ -178,24 +160,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * name is supported. * @param locale the dictionary locale. * @param dictType the dictionary type, as a human-readable string - * @param isUpdatable whether to support dynamically updating the dictionary. Please note that - * dynamic dictionary has negative effects on memory space and computation time. * @param dictFile dictionary file path. if null, use default dictionary path based on * dictionary type. */ public ExpandableBinaryDictionary(final Context context, final String dictName, - final Locale locale, final String dictType, final boolean isUpdatable, - final File dictFile) { + final Locale locale, final String dictType, final File dictFile) { super(dictType); mDictName = dictName; mContext = context; mLocale = locale; - mIsUpdatable = isUpdatable; mDictFile = getDictFile(context, dictName, dictFile); mBinaryDictionary = null; mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName); - // Currently, only dynamic personalization dictionary is updatable. - mDictionaryWriter = getDictionaryWriter(isUpdatable); } public static File getDictFile(final Context context, final String dictName, @@ -225,19 +201,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } - protected void closeBinaryDictionary() { - // Ensure that no other threads are accessing the local binary dictionary. - ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary != null) { - mBinaryDictionary.close(); - mBinaryDictionary = null; - } - } - }); - } - protected Map getHeaderAttributeMap() { HashMap attributeMap = new HashMap(); attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); @@ -257,47 +220,28 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary = null; } + private void createBinaryDictionaryLocked() { + BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); + } + + private void openBinaryDictionaryLocked() { + mBinaryDictionary = new BinaryDictionary( + mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), + true /* useFullEditDistance */, mLocale, mDictType, true /* isUpdatable */); + } + protected void clear() { ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (mDictionaryWriter == null) { - removeBinaryDictionaryLocked(); - BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); - mBinaryDictionary = new BinaryDictionary( - mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), - true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable); - } else { - mDictionaryWriter.clear(); - } + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); } }); } - /** - * Adds a word unigram to the dictionary. Used for loading a dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - protected void addWord(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord) { - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord); - } - - /** - * Adds a word bigram in the dictionary. Used for loading a dictionary. - */ - protected void addBigram(final String prevWord, final String word, final int frequency, - final long lastModifiedTime) { - mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */, - lastModifiedTime); - } - /** * Check whether GC is needed and run GC if required. */ @@ -305,13 +249,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(mindsBlockByGC); + runGCAfterAllPrioritizedTasksIfRequiredLocked(mindsBlockByGC); } }); } - private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { - // Calls to needsToRunGC() need to be serialized. + protected void runGCIfRequiredLocked(final boolean mindsBlockByGC) { + if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { + mBinaryDictionary.flushWithGC(); + } + } + + private void runGCAfterAllPrioritizedTasksIfRequiredLocked(final boolean mindsBlockByGC) { + // needsToRunGC() have to be called with lock. if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { if (setProcessingLargeTaskIfNot()) { // Run GC after currently existing time sensitive operations. @@ -335,52 +285,50 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void addWordDynamically(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - if (!mIsUpdatable) { - Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, isNotAWord, isBlacklisted, timestamp); } }); } + protected void addWordDynamicallyLocked(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp) { + mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); + } + /** * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. */ protected void addBigramDynamically(final String word0, final String word1, final int frequency, final int timestamp) { - if (!mIsUpdatable) { - Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " - + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addBigramDynamicallyLocked(word0, word1, frequency, timestamp); } }); } + protected void addBigramDynamicallyLocked(final String word0, final String word1, + final int frequency, final int timestamp) { + mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + } + /** * Dynamically remove a word bigram in the dictionary. */ protected void removeBigramDynamically(final String word0, final String word1) { - if (!mIsUpdatable) { - Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " - + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); mBinaryDictionary.removeBigramWords(word0, word1); } }); @@ -396,11 +344,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void addMultipleDictionaryEntriesDynamically( final ArrayList languageModelParams, final AddMultipleDictionaryEntriesCallback callback) { - if (!mIsUpdatable) { - Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " + - "dictionary: " + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { @@ -463,10 +406,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public boolean isValidWord(final String word) { reloadDictionaryIfRequired(); - return isValidWordInner(word); - } - - protected boolean isValidWordInner(final String word) { if (processingLargeTask()) { return false; } @@ -503,7 +442,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Loads the current binary dictionary from internal storage. Assumes the dictionary file * exists. */ - private void loadBinaryDictionary() { + private void loadBinaryDictionaryLocked() { if (DEBUG) { Log.d(TAG, "Loading binary dictionary: " + mDictName + " request=" + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" @@ -519,65 +458,40 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } catch (InterruptedException e) { } } - - final String filename = mDictFile.getAbsolutePath(); - final long length = mDictFile.length(); - - // Build the new binary dictionary - final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */, - length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - - // Ensure all threads accessing the current dictionary have finished before - // swapping in the new one. - // TODO: Ensure multi-thread assignment of mBinaryDictionary. final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; - ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - mBinaryDictionary = newBinaryDictionary; - if (oldBinaryDictionary != null) { - oldBinaryDictionary.close(); - } - } - }); + openBinaryDictionaryLocked(); + if (oldBinaryDictionary != null) { + oldBinaryDictionary.close(); + } } /** * Abstract method for checking if it is required to reload the dictionary before writing * a binary dictionary. */ - abstract protected boolean needsToReloadBeforeWriting(); + abstract protected boolean needsToReloadAfterCreation(); /** - * Writes a new binary dictionary based on the contents of the fusion dictionary. + * Create a new binary dictionary and load initial contents. */ - private void writeBinaryDictionary() { + private void createNewDictionaryLocked() { if (DEBUG) { Log.d(TAG, "Generating binary dictionary: " + mDictName + " request=" + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + mDictNameDictionaryUpdateController.mLastUpdateTime); } - if (needsToReloadBeforeWriting()) { - mDictionaryWriter.clear(); - loadDictionaryAsync(); - mDictionaryWriter.write(mDictFile, getHeaderAttributeMap()); + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); + loadInitialContentsLocked(); + mBinaryDictionary.flushWithGC(); + } + + private void flushDictionaryLocked() { + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + mBinaryDictionary.flushWithGC(); } else { - if (mBinaryDictionary == null || !isValidDictionary() - // TODO: remove the check below - || !matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion())) { - if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) { - Log.e(TAG, "Can't remove a file: " + mDictFile.getName()); - } - BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); - } else { - if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { - mBinaryDictionary.flushWithGC(); - } else { - mBinaryDictionary.flush(); - } - } + mBinaryDictionary.flush(); } } @@ -638,52 +552,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { try { final long time = System.currentTimeMillis(); - final boolean dictionaryFileExists = dictionaryFileExists(); - if (mDictNameDictionaryUpdateController.isOutOfDate() - || !dictionaryFileExists) { - // If the shared dictionary file does not exist or is out of date, the - // first instance that acquires the lock will generate a new one. - if (hasContentChanged() || !dictionaryFileExists) { - // If the source content has changed or the dictionary does not exist, - // rebuild the binary dictionary. Empty dictionaries are supported (in - // the case where loadDictionaryAsync() adds nothing) in order to - // provide a uniform framework. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } else { - // If not, the reload request was unnecessary so revert - // LastUpdateRequestTime to LastUpdateTime. - mDictNameDictionaryUpdateController.mLastUpdateRequestTime = - mDictNameDictionaryUpdateController.mLastUpdateTime; - } + final boolean openedDictIsOutOfDate = + mDictNameDictionaryUpdateController.isOutOfDate(); + if (!dictionaryFileExists() + || (openedDictIsOutOfDate && haveContentsChanged())) { + // If the shared dictionary file does not exist or is out of date and + // contents have been updated, the first instance that acquires the lock + // will generate a new one + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); + } else if (openedDictIsOutOfDate) { + // If not, the reload request was unnecessary so revert + // LastUpdateRequestTime to LastUpdateTime. + mDictNameDictionaryUpdateController.mLastUpdateRequestTime = + mDictNameDictionaryUpdateController.mLastUpdateTime; } else if (mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.mLastUpdateTime < mDictNameDictionaryUpdateController.mLastUpdateTime) { // Otherwise, if the local dictionary is older than the shared dictionary, // load the shared dictionary. - loadBinaryDictionary(); + loadBinaryDictionaryLocked(); } - // If we just loaded the binary dictionary, then mBinaryDictionary is not - // up-to-date yet so it's useless to test it right away. Schedule the check - // for right after it's loaded instead. - ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary != null && !(isValidDictionary() - // TODO: remove the check below - && matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion()))) { - // Binary dictionary or its format version is not valid. Regenerate - // the dictionary file. writeBinaryDictionary will remove the - // existing files if appropriate. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } - mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; - } - }); + if (mBinaryDictionary != null && !(isValidDictionaryLocked() + // TODO: remove the check below + && matchesExpectedBinaryDictFormatVersionForThisType( + mBinaryDictionary.getFormatVersion()))) { + // Binary dictionary or its format version is not valid. Regenerate + // the dictionary file. writeBinaryDictionary will remove the + // existing files if appropriate. + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); + } + mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; } finally { mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); } @@ -697,13 +597,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Generate binary dictionary using DictionaryWriter. + * Flush binary dictionary to dictionary file. */ protected void asyncFlushBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { - writeBinaryDictionary(); + flushDictionaryLocked(); } }; final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index 3e3cbf063..8078ab541 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -86,8 +86,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { public UserBinaryDictionary(final Context context, final Locale locale, final boolean alsoUseMoreRestrictiveLocales, final File dictFile) { - super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER, - false /* isUpdatable */, dictFile); + super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER, dictFile); if (null == locale) throw new NullPointerException(); // Catch the error earlier final String localeStr = locale.toString(); if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) { @@ -130,7 +129,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } @Override - public void loadDictionaryAsync() { + public void loadInitialContentsLocked() { // Split the locale. For example "en" => ["en"], "de_DE" => ["de", "DE"], // "en_US_foo_bar_qux" => ["en", "US", "foo_bar_qux"] because of the limit of 3. // This is correct for locale processing. @@ -182,7 +181,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { try { cursor = mContext.getContentResolver().query( Words.CONTENT_URI, PROJECTION_QUERY, request.toString(), requestArguments, null); - addWords(cursor); + addWordsLocked(cursor); } catch (final SQLiteException e) { Log.e(TAG, "SQLiteException in the remote User dictionary process.", e); } finally { @@ -236,7 +235,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } } - private void addWords(final Cursor cursor) { + private void addWordsLocked(final Cursor cursor) { final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; if (cursor == null) return; if (cursor.moveToFirst()) { @@ -250,12 +249,16 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency); // Safeguard against adding really long words. if (word.length() < MAX_WORD_LENGTH) { - super.addWord(word, null, adjustedFrequency, 0 /* shortcutFreq */, - false /* isNotAWord */); - } - if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { - super.addWord(shortcut, word, adjustedFrequency, USER_DICT_SHORTCUT_FREQUENCY, - true /* isNotAWord */); + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */, + 0 /* shortcutFreq */, false /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(shortcut, adjustedFrequency, word, + USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + } } cursor.moveToNext(); } @@ -263,12 +266,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { return true; } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return true; } } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java index b09e20591..db96de305 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java @@ -139,7 +139,7 @@ class InputLogicHandler implements Handler.Callback { forEnd /* dismissGestureFloatingPreviewText */); if (forEnd) { mInBatchInput = false; - // The following call schedules onEndBatchInputAsyncInternal + // The following call schedules onEndBatchInputInternal // to be called on the UI thread. mLatinIME.mHandler.onEndBatchInput(suggestedWords); } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java index a5dc45691..678c5ca6b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; @@ -25,6 +26,7 @@ import java.io.IOException; * An interface of binary dictionary encoder. */ public interface DictEncoder { + @UsedForTesting public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException; diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 8c358cd50..68b47315a 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -27,10 +27,8 @@ import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.util.ArrayList; -import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; /** * This class is a base class of a dictionary that supports decaying for the personalized language @@ -49,15 +47,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB /** The locale for this dictionary. */ public final Locale mLocale; - private final String mDictName; private Map mAdditionalAttributeMap = null; protected DecayingExpandableBinaryDictionaryBase(final Context context, final String dictName, final Locale locale, final String dictionaryType, final File dictFile) { - super(context, dictName, locale, dictionaryType, true /* isUpdatable */, dictFile); + super(context, dictName, locale, dictionaryType, dictFile); mLocale = locale; - mDictName = dictName; if (mLocale != null && mLocale.toString().length() > 1) { reloadDictionaryIfRequired(); } @@ -79,7 +75,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @Override protected Map getHeaderAttributeMap() { - final Map attributeMap = new HashMap(); + final Map attributeMap = super.getHeaderAttributeMap(); if (mAdditionalAttributeMap != null) { attributeMap.putAll(mAdditionalAttributeMap); } @@ -87,20 +83,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB DictionaryHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, DictionaryHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); - attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); - attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, - String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); return attributeMap; } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { return false; } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return false; } @@ -144,8 +136,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB } @Override - protected void loadDictionaryAsync() { - // Never loaded to memory in Java side. + protected void loadInitialContentsLocked() { + // No initial contents. } @UsedForTesting