Merge "Use dynamic operations to construct all ver4 dicts."
This commit is contained in:
commit
58a536e41a
9 changed files with 150 additions and 414 deletions
|
@ -1,72 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (C) 2013 The Android Open Source Project
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.android.inputmethod.latin;
|
|
||||||
|
|
||||||
import android.util.Log;
|
|
||||||
|
|
||||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
|
||||||
import com.android.inputmethod.latin.makedict.Ver4DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.utils.FileUtils;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
abstract public class AbstractDictionaryWriter {
|
|
||||||
/** Used for Log actions from this class */
|
|
||||||
private static final String TAG = AbstractDictionaryWriter.class.getSimpleName();
|
|
||||||
|
|
||||||
public AbstractDictionaryWriter() {
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract public void clear();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add a unigram with an optional shortcut to the dictionary.
|
|
||||||
* @param word The word to add.
|
|
||||||
* @param shortcutTarget A shortcut target for this word, or null if none.
|
|
||||||
* @param frequency The frequency for this unigram.
|
|
||||||
* @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
|
|
||||||
* if shortcutTarget is null.
|
|
||||||
* @param isNotAWord true if this is not a word, i.e. shortcut only.
|
|
||||||
*/
|
|
||||||
abstract public void addUnigramWord(final String word, final String shortcutTarget,
|
|
||||||
final int frequency, final int shortcutFreq, final boolean isNotAWord);
|
|
||||||
|
|
||||||
// TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve.
|
|
||||||
abstract public void addBigramWords(final String word0, final String word1,
|
|
||||||
final int frequency, final boolean isValid, final long lastModifiedTime);
|
|
||||||
|
|
||||||
abstract public void removeBigramWords(final String word0, final String word1);
|
|
||||||
|
|
||||||
abstract protected void writeDictionary(final DictEncoder dictEncoder,
|
|
||||||
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
|
|
||||||
|
|
||||||
public void write(final File file, final Map<String, String> attributeMap) {
|
|
||||||
try {
|
|
||||||
FileUtils.deleteRecursively(file);
|
|
||||||
file.mkdir();
|
|
||||||
final DictEncoder dictEncoder = new Ver4DictEncoder(file);
|
|
||||||
writeDictionary(dictEncoder, attributeMap);
|
|
||||||
} catch (IOException e) {
|
|
||||||
Log.e(TAG, "IO exception while writing file", e);
|
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
Log.e(TAG, "Unsupported format", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -31,7 +31,6 @@ import com.android.inputmethod.latin.makedict.WordProperty;
|
||||||
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
|
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
|
||||||
import com.android.inputmethod.latin.settings.NativeSuggestOptions;
|
import com.android.inputmethod.latin.settings.NativeSuggestOptions;
|
||||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||||
import com.android.inputmethod.latin.utils.FileUtils;
|
|
||||||
import com.android.inputmethod.latin.utils.JniUtils;
|
import com.android.inputmethod.latin.utils.JniUtils;
|
||||||
import com.android.inputmethod.latin.utils.LanguageModelParam;
|
import com.android.inputmethod.latin.utils.LanguageModelParam;
|
||||||
import com.android.inputmethod.latin.utils.StringUtils;
|
import com.android.inputmethod.latin.utils.StringUtils;
|
||||||
|
|
|
@ -78,7 +78,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
public ContactsBinaryDictionary(final Context context, final Locale locale,
|
public ContactsBinaryDictionary(final Context context, final Locale locale,
|
||||||
final File dictFile) {
|
final File dictFile) {
|
||||||
super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS,
|
super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS,
|
||||||
false /* isUpdatable */, dictFile);
|
dictFile);
|
||||||
mLocale = locale;
|
mLocale = locale;
|
||||||
mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);
|
mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);
|
||||||
registerObserver(context);
|
registerObserver(context);
|
||||||
|
@ -114,14 +114,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void loadDictionaryAsync() {
|
public void loadInitialContentsLocked() {
|
||||||
loadDeviceAccountsEmailAddresses();
|
loadDeviceAccountsEmailAddressesLocked();
|
||||||
loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI);
|
loadDictionaryForUriLocked(ContactsContract.Profile.CONTENT_URI);
|
||||||
// TODO: Switch this URL to the newer ContactsContract too
|
// TODO: Switch this URL to the newer ContactsContract too
|
||||||
loadDictionaryAsyncForUri(Contacts.CONTENT_URI);
|
loadDictionaryForUriLocked(Contacts.CONTENT_URI);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void loadDeviceAccountsEmailAddresses() {
|
private void loadDeviceAccountsEmailAddressesLocked() {
|
||||||
final List<String> accountVocabulary =
|
final List<String> accountVocabulary =
|
||||||
AccountUtils.getDeviceAccountsEmailAddresses(mContext);
|
AccountUtils.getDeviceAccountsEmailAddresses(mContext);
|
||||||
if (accountVocabulary == null || accountVocabulary.isEmpty()) {
|
if (accountVocabulary == null || accountVocabulary.isEmpty()) {
|
||||||
|
@ -131,12 +131,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
if (DEBUG) {
|
if (DEBUG) {
|
||||||
Log.d(TAG, "loadAccountVocabulary: " + word);
|
Log.d(TAG, "loadAccountVocabulary: " + word);
|
||||||
}
|
}
|
||||||
super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, 0 /* shortcutFreq */,
|
runGCIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
false /* isNotAWord */);
|
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */,
|
||||||
|
0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */,
|
||||||
|
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void loadDictionaryAsyncForUri(final Uri uri) {
|
private void loadDictionaryForUriLocked(final Uri uri) {
|
||||||
Cursor cursor = null;
|
Cursor cursor = null;
|
||||||
try {
|
try {
|
||||||
cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null);
|
cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null);
|
||||||
|
@ -145,7 +147,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
if (cursor.moveToFirst()) {
|
if (cursor.moveToFirst()) {
|
||||||
sContactCountAtLastRebuild = getContactCount();
|
sContactCountAtLastRebuild = getContactCount();
|
||||||
addWords(cursor);
|
addWordsLocked(cursor);
|
||||||
}
|
}
|
||||||
} catch (final SQLiteException e) {
|
} catch (final SQLiteException e) {
|
||||||
Log.e(TAG, "SQLiteException in the remote Contacts process.", e);
|
Log.e(TAG, "SQLiteException in the remote Contacts process.", e);
|
||||||
|
@ -166,12 +168,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addWords(final Cursor cursor) {
|
private void addWordsLocked(final Cursor cursor) {
|
||||||
int count = 0;
|
int count = 0;
|
||||||
while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) {
|
while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) {
|
||||||
String name = cursor.getString(INDEX_NAME);
|
String name = cursor.getString(INDEX_NAME);
|
||||||
if (isValidName(name)) {
|
if (isValidName(name)) {
|
||||||
addName(name);
|
addNameLocked(name);
|
||||||
++count;
|
++count;
|
||||||
} else {
|
} else {
|
||||||
if (DEBUG_DUMP) {
|
if (DEBUG_DUMP) {
|
||||||
|
@ -207,7 +209,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
* Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their
|
* Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their
|
||||||
* bigrams depending on locale.
|
* bigrams depending on locale.
|
||||||
*/
|
*/
|
||||||
private void addName(final String name) {
|
private void addNameLocked(final String name) {
|
||||||
int len = StringUtils.codePointCount(name);
|
int len = StringUtils.codePointCount(name);
|
||||||
String prevWord = null;
|
String prevWord = null;
|
||||||
// TODO: Better tokenization for non-Latin writing systems
|
// TODO: Better tokenization for non-Latin writing systems
|
||||||
|
@ -226,13 +228,15 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
if (DEBUG) {
|
if (DEBUG) {
|
||||||
Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);
|
Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);
|
||||||
}
|
}
|
||||||
super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS,
|
runGCIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
0 /* shortcutFreq */, false /* isNotAWord */);
|
addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS,
|
||||||
if (!TextUtils.isEmpty(prevWord)) {
|
null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */,
|
||||||
if (mUseFirstLastBigrams) {
|
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
|
||||||
super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM,
|
if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {
|
||||||
0 /* lastModifiedTime */);
|
runGCIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
}
|
addBigramDynamicallyLocked(prevWord, word,
|
||||||
|
FREQUENCY_FOR_CONTACTS_BIGRAM,
|
||||||
|
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
|
||||||
}
|
}
|
||||||
prevWord = word;
|
prevWord = word;
|
||||||
}
|
}
|
||||||
|
@ -258,12 +262,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean needsToReloadBeforeWriting() {
|
protected boolean needsToReloadAfterCreation() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean hasContentChanged() {
|
protected boolean haveContentsChanged() {
|
||||||
final long startTime = SystemClock.uptimeMillis();
|
final long startTime = SystemClock.uptimeMillis();
|
||||||
final int contactCount = getContactCount();
|
final int contactCount = getContactCount();
|
||||||
if (contactCount > MAX_CONTACT_COUNT) {
|
if (contactCount > MAX_CONTACT_COUNT) {
|
||||||
|
@ -291,7 +295,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
if (cursor.moveToFirst()) {
|
if (cursor.moveToFirst()) {
|
||||||
while (!cursor.isAfterLast()) {
|
while (!cursor.isAfterLast()) {
|
||||||
String name = cursor.getString(INDEX_NAME);
|
String name = cursor.getString(INDEX_NAME);
|
||||||
if (isValidName(name) && !isNameInDictionary(name)) {
|
if (isValidName(name) && !isNameInDictionaryLocked(name)) {
|
||||||
if (DEBUG) {
|
if (DEBUG) {
|
||||||
Log.d(TAG, "Contact name missing: " + name + " (runtime = "
|
Log.d(TAG, "Contact name missing: " + name + " (runtime = "
|
||||||
+ (SystemClock.uptimeMillis() - startTime) + " ms)");
|
+ (SystemClock.uptimeMillis() - startTime) + " ms)");
|
||||||
|
@ -321,7 +325,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
/**
|
/**
|
||||||
* Checks if the words in a name are in the current binary dictionary.
|
* Checks if the words in a name are in the current binary dictionary.
|
||||||
*/
|
*/
|
||||||
private boolean isNameInDictionary(final String name) {
|
private boolean isNameInDictionaryLocked(final String name) {
|
||||||
int len = StringUtils.codePointCount(name);
|
int len = StringUtils.codePointCount(name);
|
||||||
String prevWord = null;
|
String prevWord = null;
|
||||||
for (int i = 0; i < len; i++) {
|
for (int i = 0; i < len; i++) {
|
||||||
|
@ -332,11 +336,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
final int wordLen = StringUtils.codePointCount(word);
|
final int wordLen = StringUtils.codePointCount(word);
|
||||||
if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
|
if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
|
||||||
if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {
|
if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {
|
||||||
if (!super.isValidBigramLocked(prevWord, word)) {
|
if (!isValidBigramLocked(prevWord, word)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (!super.isValidWordLocked(word)) {
|
if (!isValidWordLocked(word)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,92 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (C) 2013 The Android Open Source Project
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.android.inputmethod.latin;
|
|
||||||
|
|
||||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
|
||||||
import com.android.inputmethod.latin.makedict.ProbabilityInfo;
|
|
||||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
|
||||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* An in memory dictionary for memorizing entries and writing a binary dictionary.
|
|
||||||
*/
|
|
||||||
public class DictionaryWriter extends AbstractDictionaryWriter {
|
|
||||||
private static final int BINARY_DICT_VERSION = FormatSpec.VERSION4;
|
|
||||||
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
|
|
||||||
new FormatSpec.FormatOptions(BINARY_DICT_VERSION, false /* hasTimestamp */);
|
|
||||||
|
|
||||||
private FusionDictionary mFusionDictionary;
|
|
||||||
|
|
||||||
public DictionaryWriter() {
|
|
||||||
clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void clear() {
|
|
||||||
final HashMap<String, String> attributes = CollectionUtils.newHashMap();
|
|
||||||
mFusionDictionary = new FusionDictionary(new PtNodeArray(),
|
|
||||||
new FusionDictionary.DictionaryOptions(attributes));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adds a word unigram to the fusion dictionary.
|
|
||||||
*/
|
|
||||||
// TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
|
|
||||||
// considering performance regression.
|
|
||||||
@Override
|
|
||||||
public void addUnigramWord(final String word, final String shortcutTarget,
|
|
||||||
final int probability, final int shortcutProbability, final boolean isNotAWord) {
|
|
||||||
if (shortcutTarget == null) {
|
|
||||||
mFusionDictionary.add(word, new ProbabilityInfo(probability), null, isNotAWord);
|
|
||||||
} else {
|
|
||||||
// TODO: Do this in the subclass, with this class taking an arraylist.
|
|
||||||
final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
|
|
||||||
shortcutTargets.add(new WeightedString(shortcutTarget, shortcutProbability));
|
|
||||||
mFusionDictionary.add(word, new ProbabilityInfo(probability), shortcutTargets,
|
|
||||||
isNotAWord);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void addBigramWords(final String word0, final String word1, final int probability,
|
|
||||||
final boolean isValid, final long lastModifiedTime) {
|
|
||||||
mFusionDictionary.setBigram(word0, word1, new ProbabilityInfo(probability));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void removeBigramWords(final String word0, final String word1) {
|
|
||||||
// This class don't support removing bigram words.
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void writeDictionary(final DictEncoder dictEncoder,
|
|
||||||
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
|
|
||||||
for (final Map.Entry<String, String> entry : attributeMap.entrySet()) {
|
|
||||||
mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue());
|
|
||||||
}
|
|
||||||
dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -32,7 +32,6 @@ import com.android.inputmethod.latin.utils.CombinedFormatUtils;
|
||||||
import com.android.inputmethod.latin.utils.ExecutorUtils;
|
import com.android.inputmethod.latin.utils.ExecutorUtils;
|
||||||
import com.android.inputmethod.latin.utils.FileUtils;
|
import com.android.inputmethod.latin.utils.FileUtils;
|
||||||
import com.android.inputmethod.latin.utils.LanguageModelParam;
|
import com.android.inputmethod.latin.utils.LanguageModelParam;
|
||||||
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
|
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -90,10 +89,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
*/
|
*/
|
||||||
private BinaryDictionary mBinaryDictionary;
|
private BinaryDictionary mBinaryDictionary;
|
||||||
|
|
||||||
// TODO: Remove and handle dictionaries in native code.
|
|
||||||
/** The in-memory dictionary used to generate the binary dictionary. */
|
|
||||||
protected AbstractDictionaryWriter mDictionaryWriter;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The name of this dictionary, used as a part of the filename for storing the binary
|
* The name of this dictionary, used as a part of the filename for storing the binary
|
||||||
* dictionary. Multiple dictionary instances with the same name is supported, with access
|
* dictionary. Multiple dictionary instances with the same name is supported, with access
|
||||||
|
@ -104,9 +99,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
/** Dictionary locale */
|
/** Dictionary locale */
|
||||||
private final Locale mLocale;
|
private final Locale mLocale;
|
||||||
|
|
||||||
/** Whether to support dynamically updating the dictionary */
|
|
||||||
private final boolean mIsUpdatable;
|
|
||||||
|
|
||||||
/** Dictionary file */
|
/** Dictionary file */
|
||||||
private final File mDictFile;
|
private final File mDictFile;
|
||||||
|
|
||||||
|
@ -126,23 +118,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
new AtomicReference<Runnable>();
|
new AtomicReference<Runnable>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
|
* Abstract method for loading initial contents of a given dictionary.
|
||||||
* thread.
|
|
||||||
*/
|
*/
|
||||||
protected abstract void loadDictionaryAsync();
|
protected abstract void loadInitialContentsLocked();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Indicates that the source dictionary content has changed and a rebuild of the binary file is
|
* Indicates that the source dictionary contents have changed and a rebuild of the binary file
|
||||||
* required. If it returns false, the next reload will only read the current binary dictionary
|
* is required. If it returns false, the next reload will only read the current binary
|
||||||
* from file. Note that the shared binary dictionary is locked when this is called.
|
* dictionary from file. Note that the shared binary dictionary is locked when this is called.
|
||||||
*/
|
*/
|
||||||
protected abstract boolean hasContentChanged();
|
protected abstract boolean haveContentsChanged();
|
||||||
|
|
||||||
private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) {
|
private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) {
|
||||||
return formatVersion == FormatSpec.VERSION4;
|
return formatVersion == FormatSpec.VERSION4;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isValidDictionary() {
|
public boolean isValidDictionaryLocked() {
|
||||||
return mBinaryDictionary.isValidDictionary();
|
return mBinaryDictionary.isValidDictionary();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -161,15 +152,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
return recorder;
|
return recorder;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static AbstractDictionaryWriter getDictionaryWriter(
|
|
||||||
final boolean isDynamicPersonalizationDictionary) {
|
|
||||||
if (isDynamicPersonalizationDictionary) {
|
|
||||||
return null;
|
|
||||||
} else {
|
|
||||||
return new DictionaryWriter();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a new expandable binary dictionary.
|
* Creates a new expandable binary dictionary.
|
||||||
*
|
*
|
||||||
|
@ -178,24 +160,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
* name is supported.
|
* name is supported.
|
||||||
* @param locale the dictionary locale.
|
* @param locale the dictionary locale.
|
||||||
* @param dictType the dictionary type, as a human-readable string
|
* @param dictType the dictionary type, as a human-readable string
|
||||||
* @param isUpdatable whether to support dynamically updating the dictionary. Please note that
|
|
||||||
* dynamic dictionary has negative effects on memory space and computation time.
|
|
||||||
* @param dictFile dictionary file path. if null, use default dictionary path based on
|
* @param dictFile dictionary file path. if null, use default dictionary path based on
|
||||||
* dictionary type.
|
* dictionary type.
|
||||||
*/
|
*/
|
||||||
public ExpandableBinaryDictionary(final Context context, final String dictName,
|
public ExpandableBinaryDictionary(final Context context, final String dictName,
|
||||||
final Locale locale, final String dictType, final boolean isUpdatable,
|
final Locale locale, final String dictType, final File dictFile) {
|
||||||
final File dictFile) {
|
|
||||||
super(dictType);
|
super(dictType);
|
||||||
mDictName = dictName;
|
mDictName = dictName;
|
||||||
mContext = context;
|
mContext = context;
|
||||||
mLocale = locale;
|
mLocale = locale;
|
||||||
mIsUpdatable = isUpdatable;
|
|
||||||
mDictFile = getDictFile(context, dictName, dictFile);
|
mDictFile = getDictFile(context, dictName, dictFile);
|
||||||
mBinaryDictionary = null;
|
mBinaryDictionary = null;
|
||||||
mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName);
|
mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName);
|
||||||
// Currently, only dynamic personalization dictionary is updatable.
|
|
||||||
mDictionaryWriter = getDictionaryWriter(isUpdatable);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static File getDictFile(final Context context, final String dictName,
|
public static File getDictFile(final Context context, final String dictName,
|
||||||
|
@ -225,19 +201,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void closeBinaryDictionary() {
|
|
||||||
// Ensure that no other threads are accessing the local binary dictionary.
|
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
if (mBinaryDictionary != null) {
|
|
||||||
mBinaryDictionary.close();
|
|
||||||
mBinaryDictionary = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
protected Map<String, String> getHeaderAttributeMap() {
|
protected Map<String, String> getHeaderAttributeMap() {
|
||||||
HashMap<String, String> attributeMap = new HashMap<String, String>();
|
HashMap<String, String> attributeMap = new HashMap<String, String>();
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName);
|
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName);
|
||||||
|
@ -257,47 +220,28 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
mBinaryDictionary = null;
|
mBinaryDictionary = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void createBinaryDictionaryLocked() {
|
||||||
|
BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
|
||||||
|
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void openBinaryDictionaryLocked() {
|
||||||
|
mBinaryDictionary = new BinaryDictionary(
|
||||||
|
mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(),
|
||||||
|
true /* useFullEditDistance */, mLocale, mDictType, true /* isUpdatable */);
|
||||||
|
}
|
||||||
|
|
||||||
protected void clear() {
|
protected void clear() {
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
if (mDictionaryWriter == null) {
|
removeBinaryDictionaryLocked();
|
||||||
removeBinaryDictionaryLocked();
|
createBinaryDictionaryLocked();
|
||||||
BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
|
openBinaryDictionaryLocked();
|
||||||
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
|
|
||||||
mBinaryDictionary = new BinaryDictionary(
|
|
||||||
mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(),
|
|
||||||
true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
|
|
||||||
} else {
|
|
||||||
mDictionaryWriter.clear();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Adds a word unigram to the dictionary. Used for loading a dictionary.
|
|
||||||
* @param word The word to add.
|
|
||||||
* @param shortcutTarget A shortcut target for this word, or null if none.
|
|
||||||
* @param frequency The frequency for this unigram.
|
|
||||||
* @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
|
|
||||||
* if shortcutTarget is null.
|
|
||||||
* @param isNotAWord true if this is not a word, i.e. shortcut only.
|
|
||||||
*/
|
|
||||||
protected void addWord(final String word, final String shortcutTarget,
|
|
||||||
final int frequency, final int shortcutFreq, final boolean isNotAWord) {
|
|
||||||
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adds a word bigram in the dictionary. Used for loading a dictionary.
|
|
||||||
*/
|
|
||||||
protected void addBigram(final String prevWord, final String word, final int frequency,
|
|
||||||
final long lastModifiedTime) {
|
|
||||||
mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */,
|
|
||||||
lastModifiedTime);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check whether GC is needed and run GC if required.
|
* Check whether GC is needed and run GC if required.
|
||||||
*/
|
*/
|
||||||
|
@ -305,13 +249,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
runGCIfRequiredInternalLocked(mindsBlockByGC);
|
runGCAfterAllPrioritizedTasksIfRequiredLocked(mindsBlockByGC);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) {
|
protected void runGCIfRequiredLocked(final boolean mindsBlockByGC) {
|
||||||
// Calls to needsToRunGC() need to be serialized.
|
if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) {
|
||||||
|
mBinaryDictionary.flushWithGC();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void runGCAfterAllPrioritizedTasksIfRequiredLocked(final boolean mindsBlockByGC) {
|
||||||
|
// needsToRunGC() have to be called with lock.
|
||||||
if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) {
|
if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) {
|
||||||
if (setProcessingLargeTaskIfNot()) {
|
if (setProcessingLargeTaskIfNot()) {
|
||||||
// Run GC after currently existing time sensitive operations.
|
// Run GC after currently existing time sensitive operations.
|
||||||
|
@ -335,52 +285,50 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
protected void addWordDynamically(final String word, final int frequency,
|
protected void addWordDynamically(final String word, final int frequency,
|
||||||
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
|
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
|
||||||
final boolean isBlacklisted, final int timestamp) {
|
final boolean isBlacklisted, final int timestamp) {
|
||||||
if (!mIsUpdatable) {
|
|
||||||
Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
|
runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq,
|
addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq,
|
||||||
isNotAWord, isBlacklisted, timestamp);
|
isNotAWord, isBlacklisted, timestamp);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected void addWordDynamicallyLocked(final String word, final int frequency,
|
||||||
|
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
|
||||||
|
final boolean isBlacklisted, final int timestamp) {
|
||||||
|
mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq,
|
||||||
|
isNotAWord, isBlacklisted, timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dynamically adds a word bigram in the dictionary. May overwrite an existing entry.
|
* Dynamically adds a word bigram in the dictionary. May overwrite an existing entry.
|
||||||
*/
|
*/
|
||||||
protected void addBigramDynamically(final String word0, final String word1,
|
protected void addBigramDynamically(final String word0, final String word1,
|
||||||
final int frequency, final int timestamp) {
|
final int frequency, final int timestamp) {
|
||||||
if (!mIsUpdatable) {
|
|
||||||
Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: "
|
|
||||||
+ mDictName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
|
runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp);
|
addBigramDynamicallyLocked(word0, word1, frequency, timestamp);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected void addBigramDynamicallyLocked(final String word0, final String word1,
|
||||||
|
final int frequency, final int timestamp) {
|
||||||
|
mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dynamically remove a word bigram in the dictionary.
|
* Dynamically remove a word bigram in the dictionary.
|
||||||
*/
|
*/
|
||||||
protected void removeBigramDynamically(final String word0, final String word1) {
|
protected void removeBigramDynamically(final String word0, final String word1) {
|
||||||
if (!mIsUpdatable) {
|
|
||||||
Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: "
|
|
||||||
+ mDictName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
|
runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
mBinaryDictionary.removeBigramWords(word0, word1);
|
mBinaryDictionary.removeBigramWords(word0, word1);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -396,11 +344,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
protected void addMultipleDictionaryEntriesDynamically(
|
protected void addMultipleDictionaryEntriesDynamically(
|
||||||
final ArrayList<LanguageModelParam> languageModelParams,
|
final ArrayList<LanguageModelParam> languageModelParams,
|
||||||
final AddMultipleDictionaryEntriesCallback callback) {
|
final AddMultipleDictionaryEntriesCallback callback) {
|
||||||
if (!mIsUpdatable) {
|
|
||||||
Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " +
|
|
||||||
"dictionary: " + mDictName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
ExecutorUtils.getExecutor(mDictName).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
|
@ -463,10 +406,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
@Override
|
@Override
|
||||||
public boolean isValidWord(final String word) {
|
public boolean isValidWord(final String word) {
|
||||||
reloadDictionaryIfRequired();
|
reloadDictionaryIfRequired();
|
||||||
return isValidWordInner(word);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected boolean isValidWordInner(final String word) {
|
|
||||||
if (processingLargeTask()) {
|
if (processingLargeTask()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -503,7 +442,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
* Loads the current binary dictionary from internal storage. Assumes the dictionary file
|
* Loads the current binary dictionary from internal storage. Assumes the dictionary file
|
||||||
* exists.
|
* exists.
|
||||||
*/
|
*/
|
||||||
private void loadBinaryDictionary() {
|
private void loadBinaryDictionaryLocked() {
|
||||||
if (DEBUG) {
|
if (DEBUG) {
|
||||||
Log.d(TAG, "Loading binary dictionary: " + mDictName + " request="
|
Log.d(TAG, "Loading binary dictionary: " + mDictName + " request="
|
||||||
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
|
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
|
||||||
|
@ -519,65 +458,40 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
final String filename = mDictFile.getAbsolutePath();
|
|
||||||
final long length = mDictFile.length();
|
|
||||||
|
|
||||||
// Build the new binary dictionary
|
|
||||||
final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */,
|
|
||||||
length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
|
|
||||||
|
|
||||||
// Ensure all threads accessing the current dictionary have finished before
|
|
||||||
// swapping in the new one.
|
|
||||||
// TODO: Ensure multi-thread assignment of mBinaryDictionary.
|
|
||||||
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
|
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
|
||||||
ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() {
|
openBinaryDictionaryLocked();
|
||||||
@Override
|
if (oldBinaryDictionary != null) {
|
||||||
public void run() {
|
oldBinaryDictionary.close();
|
||||||
mBinaryDictionary = newBinaryDictionary;
|
}
|
||||||
if (oldBinaryDictionary != null) {
|
|
||||||
oldBinaryDictionary.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abstract method for checking if it is required to reload the dictionary before writing
|
* Abstract method for checking if it is required to reload the dictionary before writing
|
||||||
* a binary dictionary.
|
* a binary dictionary.
|
||||||
*/
|
*/
|
||||||
abstract protected boolean needsToReloadBeforeWriting();
|
abstract protected boolean needsToReloadAfterCreation();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes a new binary dictionary based on the contents of the fusion dictionary.
|
* Create a new binary dictionary and load initial contents.
|
||||||
*/
|
*/
|
||||||
private void writeBinaryDictionary() {
|
private void createNewDictionaryLocked() {
|
||||||
if (DEBUG) {
|
if (DEBUG) {
|
||||||
Log.d(TAG, "Generating binary dictionary: " + mDictName + " request="
|
Log.d(TAG, "Generating binary dictionary: " + mDictName + " request="
|
||||||
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
|
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
|
||||||
+ mDictNameDictionaryUpdateController.mLastUpdateTime);
|
+ mDictNameDictionaryUpdateController.mLastUpdateTime);
|
||||||
}
|
}
|
||||||
if (needsToReloadBeforeWriting()) {
|
removeBinaryDictionaryLocked();
|
||||||
mDictionaryWriter.clear();
|
createBinaryDictionaryLocked();
|
||||||
loadDictionaryAsync();
|
openBinaryDictionaryLocked();
|
||||||
mDictionaryWriter.write(mDictFile, getHeaderAttributeMap());
|
loadInitialContentsLocked();
|
||||||
|
mBinaryDictionary.flushWithGC();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void flushDictionaryLocked() {
|
||||||
|
if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
|
||||||
|
mBinaryDictionary.flushWithGC();
|
||||||
} else {
|
} else {
|
||||||
if (mBinaryDictionary == null || !isValidDictionary()
|
mBinaryDictionary.flush();
|
||||||
// TODO: remove the check below
|
|
||||||
|| !matchesExpectedBinaryDictFormatVersionForThisType(
|
|
||||||
mBinaryDictionary.getFormatVersion())) {
|
|
||||||
if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) {
|
|
||||||
Log.e(TAG, "Can't remove a file: " + mDictFile.getName());
|
|
||||||
}
|
|
||||||
BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
|
|
||||||
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
|
|
||||||
} else {
|
|
||||||
if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
|
|
||||||
mBinaryDictionary.flushWithGC();
|
|
||||||
} else {
|
|
||||||
mBinaryDictionary.flush();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -638,52 +552,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
try {
|
||||||
final long time = System.currentTimeMillis();
|
final long time = System.currentTimeMillis();
|
||||||
final boolean dictionaryFileExists = dictionaryFileExists();
|
final boolean openedDictIsOutOfDate =
|
||||||
if (mDictNameDictionaryUpdateController.isOutOfDate()
|
mDictNameDictionaryUpdateController.isOutOfDate();
|
||||||
|| !dictionaryFileExists) {
|
if (!dictionaryFileExists()
|
||||||
// If the shared dictionary file does not exist or is out of date, the
|
|| (openedDictIsOutOfDate && haveContentsChanged())) {
|
||||||
// first instance that acquires the lock will generate a new one.
|
// If the shared dictionary file does not exist or is out of date and
|
||||||
if (hasContentChanged() || !dictionaryFileExists) {
|
// contents have been updated, the first instance that acquires the lock
|
||||||
// If the source content has changed or the dictionary does not exist,
|
// will generate a new one
|
||||||
// rebuild the binary dictionary. Empty dictionaries are supported (in
|
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
|
||||||
// the case where loadDictionaryAsync() adds nothing) in order to
|
createNewDictionaryLocked();
|
||||||
// provide a uniform framework.
|
} else if (openedDictIsOutOfDate) {
|
||||||
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
|
// If not, the reload request was unnecessary so revert
|
||||||
writeBinaryDictionary();
|
// LastUpdateRequestTime to LastUpdateTime.
|
||||||
loadBinaryDictionary();
|
mDictNameDictionaryUpdateController.mLastUpdateRequestTime =
|
||||||
} else {
|
mDictNameDictionaryUpdateController.mLastUpdateTime;
|
||||||
// If not, the reload request was unnecessary so revert
|
|
||||||
// LastUpdateRequestTime to LastUpdateTime.
|
|
||||||
mDictNameDictionaryUpdateController.mLastUpdateRequestTime =
|
|
||||||
mDictNameDictionaryUpdateController.mLastUpdateTime;
|
|
||||||
}
|
|
||||||
} else if (mBinaryDictionary == null ||
|
} else if (mBinaryDictionary == null ||
|
||||||
mPerInstanceDictionaryUpdateController.mLastUpdateTime
|
mPerInstanceDictionaryUpdateController.mLastUpdateTime
|
||||||
< mDictNameDictionaryUpdateController.mLastUpdateTime) {
|
< mDictNameDictionaryUpdateController.mLastUpdateTime) {
|
||||||
// Otherwise, if the local dictionary is older than the shared dictionary,
|
// Otherwise, if the local dictionary is older than the shared dictionary,
|
||||||
// load the shared dictionary.
|
// load the shared dictionary.
|
||||||
loadBinaryDictionary();
|
loadBinaryDictionaryLocked();
|
||||||
}
|
}
|
||||||
// If we just loaded the binary dictionary, then mBinaryDictionary is not
|
if (mBinaryDictionary != null && !(isValidDictionaryLocked()
|
||||||
// up-to-date yet so it's useless to test it right away. Schedule the check
|
// TODO: remove the check below
|
||||||
// for right after it's loaded instead.
|
&& matchesExpectedBinaryDictFormatVersionForThisType(
|
||||||
ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() {
|
mBinaryDictionary.getFormatVersion()))) {
|
||||||
@Override
|
// Binary dictionary or its format version is not valid. Regenerate
|
||||||
public void run() {
|
// the dictionary file. writeBinaryDictionary will remove the
|
||||||
if (mBinaryDictionary != null && !(isValidDictionary()
|
// existing files if appropriate.
|
||||||
// TODO: remove the check below
|
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
|
||||||
&& matchesExpectedBinaryDictFormatVersionForThisType(
|
createNewDictionaryLocked();
|
||||||
mBinaryDictionary.getFormatVersion()))) {
|
}
|
||||||
// Binary dictionary or its format version is not valid. Regenerate
|
mPerInstanceDictionaryUpdateController.mLastUpdateTime = time;
|
||||||
// the dictionary file. writeBinaryDictionary will remove the
|
|
||||||
// existing files if appropriate.
|
|
||||||
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
|
|
||||||
writeBinaryDictionary();
|
|
||||||
loadBinaryDictionary();
|
|
||||||
}
|
|
||||||
mPerInstanceDictionaryUpdateController.mLastUpdateTime = time;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} finally {
|
} finally {
|
||||||
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
|
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
|
||||||
}
|
}
|
||||||
|
@ -697,13 +597,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate binary dictionary using DictionaryWriter.
|
* Flush binary dictionary to dictionary file.
|
||||||
*/
|
*/
|
||||||
protected void asyncFlushBinaryDictionary() {
|
protected void asyncFlushBinaryDictionary() {
|
||||||
final Runnable newTask = new Runnable() {
|
final Runnable newTask = new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
writeBinaryDictionary();
|
flushDictionaryLocked();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask);
|
final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask);
|
||||||
|
|
|
@ -86,8 +86,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
|
|
||||||
public UserBinaryDictionary(final Context context, final Locale locale,
|
public UserBinaryDictionary(final Context context, final Locale locale,
|
||||||
final boolean alsoUseMoreRestrictiveLocales, final File dictFile) {
|
final boolean alsoUseMoreRestrictiveLocales, final File dictFile) {
|
||||||
super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER,
|
super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER, dictFile);
|
||||||
false /* isUpdatable */, dictFile);
|
|
||||||
if (null == locale) throw new NullPointerException(); // Catch the error earlier
|
if (null == locale) throw new NullPointerException(); // Catch the error earlier
|
||||||
final String localeStr = locale.toString();
|
final String localeStr = locale.toString();
|
||||||
if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) {
|
if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) {
|
||||||
|
@ -130,7 +129,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void loadDictionaryAsync() {
|
public void loadInitialContentsLocked() {
|
||||||
// Split the locale. For example "en" => ["en"], "de_DE" => ["de", "DE"],
|
// Split the locale. For example "en" => ["en"], "de_DE" => ["de", "DE"],
|
||||||
// "en_US_foo_bar_qux" => ["en", "US", "foo_bar_qux"] because of the limit of 3.
|
// "en_US_foo_bar_qux" => ["en", "US", "foo_bar_qux"] because of the limit of 3.
|
||||||
// This is correct for locale processing.
|
// This is correct for locale processing.
|
||||||
|
@ -182,7 +181,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
try {
|
try {
|
||||||
cursor = mContext.getContentResolver().query(
|
cursor = mContext.getContentResolver().query(
|
||||||
Words.CONTENT_URI, PROJECTION_QUERY, request.toString(), requestArguments, null);
|
Words.CONTENT_URI, PROJECTION_QUERY, request.toString(), requestArguments, null);
|
||||||
addWords(cursor);
|
addWordsLocked(cursor);
|
||||||
} catch (final SQLiteException e) {
|
} catch (final SQLiteException e) {
|
||||||
Log.e(TAG, "SQLiteException in the remote User dictionary process.", e);
|
Log.e(TAG, "SQLiteException in the remote User dictionary process.", e);
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -236,7 +235,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addWords(final Cursor cursor) {
|
private void addWordsLocked(final Cursor cursor) {
|
||||||
final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN;
|
final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN;
|
||||||
if (cursor == null) return;
|
if (cursor == null) return;
|
||||||
if (cursor.moveToFirst()) {
|
if (cursor.moveToFirst()) {
|
||||||
|
@ -250,12 +249,16 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency);
|
final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency);
|
||||||
// Safeguard against adding really long words.
|
// Safeguard against adding really long words.
|
||||||
if (word.length() < MAX_WORD_LENGTH) {
|
if (word.length() < MAX_WORD_LENGTH) {
|
||||||
super.addWord(word, null, adjustedFrequency, 0 /* shortcutFreq */,
|
runGCIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
false /* isNotAWord */);
|
addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */,
|
||||||
}
|
0 /* shortcutFreq */, false /* isNotAWord */,
|
||||||
if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
|
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
|
||||||
super.addWord(shortcut, word, adjustedFrequency, USER_DICT_SHORTCUT_FREQUENCY,
|
if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
|
||||||
true /* isNotAWord */);
|
runGCIfRequiredLocked(true /* mindsBlockByGC */);
|
||||||
|
addWordDynamicallyLocked(shortcut, adjustedFrequency, word,
|
||||||
|
USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */,
|
||||||
|
false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
cursor.moveToNext();
|
cursor.moveToNext();
|
||||||
}
|
}
|
||||||
|
@ -263,12 +266,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean hasContentChanged() {
|
protected boolean haveContentsChanged() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean needsToReloadBeforeWriting() {
|
protected boolean needsToReloadAfterCreation() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -139,7 +139,7 @@ class InputLogicHandler implements Handler.Callback {
|
||||||
forEnd /* dismissGestureFloatingPreviewText */);
|
forEnd /* dismissGestureFloatingPreviewText */);
|
||||||
if (forEnd) {
|
if (forEnd) {
|
||||||
mInBatchInput = false;
|
mInBatchInput = false;
|
||||||
// The following call schedules onEndBatchInputAsyncInternal
|
// The following call schedules onEndBatchInputInternal
|
||||||
// to be called on the UI thread.
|
// to be called on the UI thread.
|
||||||
mLatinIME.mHandler.onEndBatchInput(suggestedWords);
|
mLatinIME.mHandler.onEndBatchInput(suggestedWords);
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,6 +16,7 @@
|
||||||
|
|
||||||
package com.android.inputmethod.latin.makedict;
|
package com.android.inputmethod.latin.makedict;
|
||||||
|
|
||||||
|
import com.android.inputmethod.annotations.UsedForTesting;
|
||||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
|
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
|
||||||
|
|
||||||
|
@ -25,6 +26,7 @@ import java.io.IOException;
|
||||||
* An interface of binary dictionary encoder.
|
* An interface of binary dictionary encoder.
|
||||||
*/
|
*/
|
||||||
public interface DictEncoder {
|
public interface DictEncoder {
|
||||||
|
@UsedForTesting
|
||||||
public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
|
public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions)
|
||||||
throws IOException, UnsupportedFormatException;
|
throws IOException, UnsupportedFormatException;
|
||||||
|
|
||||||
|
|
|
@ -27,10 +27,8 @@ import com.android.inputmethod.latin.utils.LanguageModelParam;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class is a base class of a dictionary that supports decaying for the personalized language
|
* This class is a base class of a dictionary that supports decaying for the personalized language
|
||||||
|
@ -49,15 +47,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
||||||
/** The locale for this dictionary. */
|
/** The locale for this dictionary. */
|
||||||
public final Locale mLocale;
|
public final Locale mLocale;
|
||||||
|
|
||||||
private final String mDictName;
|
|
||||||
private Map<String, String> mAdditionalAttributeMap = null;
|
private Map<String, String> mAdditionalAttributeMap = null;
|
||||||
|
|
||||||
protected DecayingExpandableBinaryDictionaryBase(final Context context,
|
protected DecayingExpandableBinaryDictionaryBase(final Context context,
|
||||||
final String dictName, final Locale locale, final String dictionaryType,
|
final String dictName, final Locale locale, final String dictionaryType,
|
||||||
final File dictFile) {
|
final File dictFile) {
|
||||||
super(context, dictName, locale, dictionaryType, true /* isUpdatable */, dictFile);
|
super(context, dictName, locale, dictionaryType, dictFile);
|
||||||
mLocale = locale;
|
mLocale = locale;
|
||||||
mDictName = dictName;
|
|
||||||
if (mLocale != null && mLocale.toString().length() > 1) {
|
if (mLocale != null && mLocale.toString().length() > 1) {
|
||||||
reloadDictionaryIfRequired();
|
reloadDictionaryIfRequired();
|
||||||
}
|
}
|
||||||
|
@ -79,7 +75,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Map<String, String> getHeaderAttributeMap() {
|
protected Map<String, String> getHeaderAttributeMap() {
|
||||||
final Map<String, String> attributeMap = new HashMap<String, String>();
|
final Map<String, String> attributeMap = super.getHeaderAttributeMap();
|
||||||
if (mAdditionalAttributeMap != null) {
|
if (mAdditionalAttributeMap != null) {
|
||||||
attributeMap.putAll(mAdditionalAttributeMap);
|
attributeMap.putAll(mAdditionalAttributeMap);
|
||||||
}
|
}
|
||||||
|
@ -87,20 +83,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
||||||
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
|
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
|
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
|
||||||
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
|
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName);
|
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString());
|
|
||||||
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
|
|
||||||
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
|
|
||||||
return attributeMap;
|
return attributeMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean hasContentChanged() {
|
protected boolean haveContentsChanged() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean needsToReloadBeforeWriting() {
|
protected boolean needsToReloadAfterCreation() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -144,8 +136,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void loadDictionaryAsync() {
|
protected void loadInitialContentsLocked() {
|
||||||
// Never loaded to memory in Java side.
|
// No initial contents.
|
||||||
}
|
}
|
||||||
|
|
||||||
@UsedForTesting
|
@UsedForTesting
|
||||||
|
|
Loading…
Reference in a new issue