nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Make DynamicPred...Base extend ExpandableBinaryDictionary."

main
Keisuke Kuroyanagi 2013-08-26 09:56:47 +00:00 committed by Android (Google) Code Review
parent d5a3c593db 6e04d65932
commit 78ab1bb661
11 changed files with 157 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -29,6 +29,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
@ -92,6 +93,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/* A extension for a binary dictionary file. */
public static final String DICT_FILE_EXTENSION = ".dict";
private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask =
new AtomicReference<AsyncWriteBinaryDictionaryTask>();
/**
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
* thread.
@ -180,6 +184,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
protected void clear() {
mLocalDictionaryController.writeLock().lock();
try {
mDictionaryWriter.clear();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
/**
* Adds a word unigram to the dictionary. Used for loading a dictionary.
*/
@ -267,7 +280,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
if (mBinaryDictionary != null) {
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
@ -276,7 +290,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
} else if (binarySuggestion == null) {
return inMemDictSuggestion;
} else {
binarySuggestion.addAll(binarySuggestion);
binarySuggestion.addAll(inMemDictSuggestion);
return binarySuggestion;
}
} else {
@ -402,7 +416,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Reloads the dictionary if required. Reload will occur asynchronously in a separate thread.
*/
void asyncReloadDictionaryIfRequired() {
public void asyncReloadDictionaryIfRequired() {
if (!isReloadRequired()) return;
if (DEBUG) {
Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename);
@ -413,7 +427,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Reloads the dictionary if required.
*/
protected final void syncReloadDictionaryIfRequired() {
public final void syncReloadDictionaryIfRequired() {
if (!isReloadRequired()) return;
syncReloadDictionaryInternal();
}
@ -492,6 +506,68 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
/**
* Load the dictionary to memory.
*/
protected void asyncLoadDictionaryToMemory() {
new AsyncLoadDictionaryToMemoryTask().start();
}
/**
* Thread class for asynchronously loading dictionary to memory.
*/
private class AsyncLoadDictionaryToMemoryTask extends Thread {
@Override
public void run() {
mLocalDictionaryController.writeLock().lock();
try {
mSharedDictionaryController.readLock().lock();
try {
loadDictionaryAsync();
} finally {
mSharedDictionaryController.readLock().unlock();
}
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
}
/**
* Generate binary dictionary using DictionaryWriter.
*/
protected void asyncWriteBinaryDictionary() {
final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask();
newTask.start();
final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask);
if (oldTask != null) {
oldTask.interrupt();
}
}
/**
* Thread class for asynchronously writing the binary dictionary.
*/
private class AsyncWriteBinaryDictionaryTask extends Thread {
@Override
public void run() {
mSharedDictionaryController.writeLock().lock();
try {
mLocalDictionaryController.writeLock().lock();
try {
if (isInterrupted()) {
return;
}
writeBinaryDictionary();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
} finally {
mSharedDictionaryController.writeLock().unlock();
}
}
}
/**
* Lock for controlling access to a given binary dictionary and for tracking whether the
* dictionary is out of date. Can be shared across multiple dictionary instances that access the

85
java/src/com/android/inputmethod/latin/ExpandableDictionary.java
View File

@ -16,7 +16,6 @@
package com.android.inputmethod.latin;
import android.content.Context;
import android.text.TextUtils;
import android.util.Log;
@ -30,9 +29,10 @@ import java.util.ArrayList;
import java.util.LinkedList;
/**
* Base class for an in-memory dictionary that can grow dynamically and can
* Class for an in-memory dictionary that can grow dynamically and can
* be searched for suggestions and valid words.
*/
// TODO: Remove after binary dictionary supports dynamic update.
public class ExpandableDictionary extends Dictionary {
private static final String TAG = ExpandableDictionary.class.getSimpleName();
/**
@ -40,23 +40,11 @@ public class ExpandableDictionary extends Dictionary {
*/
private static final int FULL_WORD_SCORE_MULTIPLIER = 2;
// Bigram frequency is a fixed point number with 1 meaning 1.2 and 255 meaning 1.8.
protected static final int BIGRAM_MAX_FREQUENCY = 255;
private Context mContext;
private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH];
private int mMaxDepth;
private int mInputLength;
private boolean mRequiresReload;
private boolean mUpdatingDictionary;
// Use this lock before touching mUpdatingDictionary & mRequiresDownload
private Object mUpdatingLock = new Object();
private static final class Node {
Node() {}
char mCode;
int mFrequency;
boolean mTerminal;
@ -158,46 +146,12 @@ public class ExpandableDictionary extends Dictionary {
private int[][] mCodes;
public ExpandableDictionary(final Context context, final String dictType) {
public ExpandableDictionary(final String dictType) {
super(dictType);
mContext = context;
clearDictionary();
mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][];
}
public void loadDictionary() {
synchronized (mUpdatingLock) {
startDictionaryLoadingTaskLocked();
}
}
public void startDictionaryLoadingTaskLocked() {
if (!mUpdatingDictionary) {
mUpdatingDictionary = true;
mRequiresReload = false;
new LoadDictionaryTask().start();
}
}
public void setRequiresReload(final boolean reload) {
synchronized (mUpdatingLock) {
mRequiresReload = reload;
}
}
public boolean getRequiresReload() {
return mRequiresReload;
}
/** Override to load your dictionary here, on a background thread. */
public void loadDictionaryAsync() {
// empty base implementation
}
public Context getContext() {
return mContext;
}
public int getMaxWordLength() {
return Constants.DICTIONARY_MAX_WORD_LENGTH;
}
@ -257,7 +211,6 @@ public class ExpandableDictionary extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo,
final boolean blockOffensiveWords) {
if (reloadDictionaryIfRequired()) return null;
if (composer.size() > 1) {
if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) {
return null;
@ -273,17 +226,7 @@ public class ExpandableDictionary extends Dictionary {
}
}
// This reloads the dictionary if required, and returns whether it's currently updating its
// contents or not.
private boolean reloadDictionaryIfRequired() {
synchronized (mUpdatingLock) {
// If we need to update, start off a background task
if (mRequiresReload) startDictionaryLoadingTaskLocked();
return mUpdatingDictionary;
}
}
protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
final String prevWordForBigrams, final ProximityInfo proximityInfo) {
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
mInputLength = codes.size();
@ -313,11 +256,6 @@ public class ExpandableDictionary extends Dictionary {
@Override
public synchronized boolean isValidWord(final String word) {
synchronized (mUpdatingLock) {
// If we need to update, start off a background task
if (mRequiresReload) startDictionaryLoadingTaskLocked();
if (mUpdatingDictionary) return false;
}
final Node node = searchNode(mRoots, word, 0, word.length());
// If node is null, we didn't find the word, so it's not valid.
// If node.mShortcutOnly is true, then it exists as a shortcut but not as a word,
@ -353,7 +291,7 @@ public class ExpandableDictionary extends Dictionary {
* Returns the word's frequency or -1 if not found
*/
@UsedForTesting
protected int getWordFrequency(final String word) {
public int getWordFrequency(final String word) {
// Case-sensitive search
final Node node = searchNode(mRoots, word, 0, word.length());
return (node == null) ? -1 : node.mFrequency;
@ -442,7 +380,7 @@ public class ExpandableDictionary extends Dictionary {
* @param suggestions the list in which to add suggestions
*/
// TODO: Share this routine with the native code for BinaryDictionary
protected void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
final int depth, final boolean completion, final int snr, final int inputIndex,
final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) {
final int count = roots.mLength;
@ -704,17 +642,6 @@ public class ExpandableDictionary extends Dictionary {
mRoots = new NodeArray();
}
private final class LoadDictionaryTask extends Thread {
LoadDictionaryTask() {}
@Override
public void run() {
loadDictionaryAsync();
synchronized (mUpdatingLock) {
mUpdatingDictionary = false;
}
}
}
private static char toLowerCase(final char c) {
char baseChar = c;
if (c < BASE_CHARS.length) {

8
java/src/com/android/inputmethod/latin/LatinIME.java
View File

@ -74,8 +74,8 @@ import com.android.inputmethod.keyboard.MainKeyboardView;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.define.ProductionFlag;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper;
import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
import com.android.inputmethod.latin.settings.Settings;
@ -566,13 +566,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper
mUserHistoryPredictionDictionary = PersonalizationHelper
.getUserHistoryPredictionDictionary(this, localeStr, prefs);
newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
mPersonalizationDictionary = PersonalizationDictionaryHelper
mPersonalizationDictionary = PersonalizationHelper
.getPersonalizationDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
mPersonalizationPredictionDictionary = PersonalizationDictionaryHelper
mPersonalizationPredictionDictionary = PersonalizationHelper
.getPersonalizationPredictionDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary);

2
java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
View File

@ -55,7 +55,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr
public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
super(context, dictType);
mExpandableDictionary = new ExpandableDictionary(context, dictType);
mExpandableDictionary = new ExpandableDictionary(dictType);
}
@Override

297
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -18,108 +18,88 @@ package com.android.inputmethod.latin.personalization;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.AsyncTask;
import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
/**
* This class is a base class of a dictionary for the personalized prediction language model.
*/
public abstract class DynamicPredictionDictionaryBase extends ExpandableDictionary {
public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
public static final boolean DBG_SAVE_RESTORE = false;
private static final boolean DBG_STRESS_TEST = false;
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
private static final FormatOptions VERSION3 = new FormatOptions(3,
true /* supportsDynamicUpdate */);
/** Any pair being typed or picked */
private static final int FREQUENCY_FOR_TYPED = 2;
/** Maximum number of pairs. Pruning will start when databases goes above this number. */
private static final int MAX_HISTORY_BIGRAMS = 10000;
public static final int FREQUENCY_FOR_TYPED = 2;
/** Locale for which this user history dictionary is storing words */
private final String mLocale;
private final UserHistoryDictionaryBigramList mBigramList =
new UserHistoryDictionaryBigramList();
private final ReentrantLock mBigramListLock = new ReentrantLock();
private final String mFileName;
private final SharedPreferences mPrefs;
private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
CollectionUtils.newArrayList();
private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask;
// Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false;
/* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
final SharedPreferences sp, final String dictionaryType) {
super(context, dictionaryType);
final SharedPreferences sp, final String dictionaryType, final String fileName) {
super(context, locale, dictionaryType, true);
mLocale = locale;
mFileName = fileName;
mPrefs = sp;
mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>();
if (mLocale != null && mLocale.length() > 1) {
loadDictionary();
asyncLoadDictionaryToMemory();
asyncReloadDictionaryIfRequired();
}
}
@Override
public void close() {
flushPendingWrites();
// Don't close the database as locale changes will require it to be reopened anyway
// Also, the database is written to somewhat frequently, so it needs to be kept alive
// throughout the life of the process.
// mOpenHelper.close();
// Ignore close because we cache PersonalizationPredictionDictionary for each language.
// See getInstance() above.
// Close only binary dictionary to reuse this dictionary.
// super.close();
closeBinaryDictionary();
// Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary.
asyncWriteBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
}
@Override
protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo) {
// Inhibit suggestions (not predictions) for user history for now. Removing this method
// is enough to use it through the standard ExpandableDictionary way.
return null;
protected boolean hasContentChanged() {
return false;
}
@Override
protected boolean needsToReloadBeforeWriting() {
return false;
}
/**
* Return whether the passed charsequence is in the dictionary.
*/
@Override
public synchronized boolean isValidWord(final String word) {
// TODO: figure out what is the correct thing to do here.
public boolean isValidWord(final String word) {
// Words included only in the user history should be treated as not in dictionary words.
return false;
}
@ -131,74 +111,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
public int addToPersonalizationPredictionDictionary(
final String word1, final String word2, final boolean isValid) {
if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return -1;
public void addToPersonalizationPredictionDictionary(
final String word0, final String word1, final boolean isValid) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
}
if (mBigramListLock.tryLock()) {
try {
super.addWord(
word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED);
mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED);
addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED,
false /* isNotAWord */);
// Do not insert a word as a bigram of itself
if (word2.equals(word1)) {
return 0;
if (word1.equals(word0)) {
return;
}
final int freq;
if (null == word1) {
freq = FREQUENCY_FOR_TYPED;
} else {
freq = super.setBigramAndGetFrequency(
word1, word2, new ForgettingCurveParams(isValid));
if (null != word0) {
addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid);
}
mBigramList.addBigram(word1, word2);
return freq;
} finally {
mBigramListLock.unlock();
}
}
return -1;
}
public boolean cancelAddingUserHistory(final String word1, final String word2) {
if (mBigramListLock.tryLock()) {
try {
if (mBigramList.removeBigram(word1, word2)) {
return super.removeBigram(word1, word2);
}
} finally {
mBigramListLock.unlock();
}
}
return false;
}
/**
* Schedules a background thread to write any pending words to the database.
*/
private void flushPendingWrites() {
// Create a background thread to write the pending entries
final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask(
mBigramList, mLocale, this, mPrefs, getContext()).execute());
if (old != null) {
old.cancel(false);
}
public void cancelAddingUserHistory(final String word0, final String word1) {
removeBigramDynamically(word0, word1);
}
@Override
public final void loadDictionaryAsync() {
// This must be run on non-main thread
mBigramListLock.lock();
try {
loadDictionaryAsyncLocked();
} finally {
mBigramListLock.unlock();
}
}
private void loadDictionaryAsyncLocked() {
protected void loadDictionaryAsync() {
final int[] profTotalCount = { 0 };
final String locale = getLocale();
if (DBG_STRESS_TEST) {
@ -210,10 +145,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
}
}
final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
final boolean initializing = last == 0;
final long now = System.currentTimeMillis();
final String fileName = getDictionaryFileName();
final ExpandableDictionary dictionary = this;
final ExpandableBinaryDictionary dictionary = this;
final OnAddWordListener listener = new OnAddWordListener() {
@Override
public void setUnigram(final String word, final String shortcutTarget,
@ -221,29 +154,25 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "load unigram: " + word + "," + frequency);
}
dictionary.addWord(word, shortcutTarget, frequency);
addWord(word, shortcutTarget, frequency, false /* isNotAWord */);
++profTotalCount[0];
addToBigramListLocked(null, word, (byte)frequency);
}
@Override
public void setBigram(final String word1, final String word2, final int frequency) {
if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
&& word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
public void setBigram(final String word0, final String word1, final int frequency) {
if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
&& word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency);
Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
}
++profTotalCount[0];
dictionary.setBigramAndGetFrequency(
word1, word2, initializing ? new ForgettingCurveParams(true)
: new ForgettingCurveParams(frequency, now, last));
addBigram(word0, word1, frequency, last);
}
addToBigramListLocked(word1, word2, (byte)frequency);
}
};
// Load the dictionary from binary file
final File dictFile = new File(getContext().getFilesDir(), fileName);
final File dictFile = new File(mContext.getFilesDir(), mFileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
try {
@ -263,131 +192,14 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
}
}
protected abstract String getDictionaryFileName();
protected String getLocale() {
return mLocale;
}
private void addToBigramListLocked(String word0, String word1, byte fcValue) {
mBigramList.addBigram(word0, word1, fcValue);
}
/**
* Async task to write pending words to the binarydicts.
*/
private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void>
implements BigramDictionaryInterface {
private final UserHistoryDictionaryBigramList mBigramList;
private final boolean mAddLevel0Bigrams;
private final String mLocale;
private final DynamicPredictionDictionaryBase mDynamicPredictionDictionary;
private final SharedPreferences mPrefs;
private final Context mContext;
public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites,
final String locale, final DynamicPredictionDictionaryBase dict,
final SharedPreferences prefs, final Context context) {
mBigramList = pendingWrites;
mLocale = locale;
mDynamicPredictionDictionary = dict;
mPrefs = prefs;
mContext = context;
mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS;
}
@Override
protected Void doInBackground(final Void... v) {
if (isCancelled()) return null;
if (mDynamicPredictionDictionary.mIsTest) {
// If mIsTest == true, wait until the lock is released.
mDynamicPredictionDictionary.mBigramListLock.lock();
try {
doWriteTaskLocked();
} finally {
mDynamicPredictionDictionary.mBigramListLock.unlock();
}
} else if (mDynamicPredictionDictionary.mBigramListLock.tryLock()) {
try {
doWriteTaskLocked();
} finally {
mDynamicPredictionDictionary.mBigramListLock.unlock();
}
}
return null;
}
private void doWriteTaskLocked() {
if (isCancelled()) return;
mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null);
if (DBG_STRESS_TEST) {
try {
Log.w(TAG, "Start stress in closing: " + mLocale);
Thread.sleep(15000);
Log.w(TAG, "End stress in closing");
} catch (InterruptedException e) {
Log.e(TAG, "In stress test", e);
}
}
final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
final String fileName =
mDynamicPredictionDictionary.getDictionaryFileName();
final File file = new File(mContext.getFilesDir(), fileName);
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, mBigramList, VERSION3);
// Save the timestamp after we finish writing the binary dictionary.
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms.");
}
}
@Override
public int getFrequency(final String word1, final String word2) {
final int freq;
if (word1 == null) { // unigram
freq = FREQUENCY_FOR_TYPED;
final byte prevFc = mBigramList.getBigrams(word1).get(word2);
} else { // bigram
final NextWord nw =
mDynamicPredictionDictionary.getBigramWord(word1, word2);
if (nw != null) {
final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = mBigramList.getBigrams(word1).get(word2);
final byte fc = fcp.getFc();
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
freq = fc & 0xFF;
} else if (UserHistoryForgettingCurveUtils.
needsToSave(fc, isValid, mAddLevel0Bigrams)) {
freq = fc & 0xFF;
} else {
// Delete this entry
freq = -1;
}
} else {
// Delete this entry
freq = -1;
}
}
return freq;
}
}
@UsedForTesting
/* package for test */ void forceAddWordForTest(
final String word1, final String word2, final boolean isValid) {
mBigramListLock.lock();
try {
addToPersonalizationPredictionDictionary(word1, word2, isValid);
} finally {
mBigramListLock.unlock();
}
final String word0, final String word1, final boolean isValid) {
addToPersonalizationPredictionDictionary(word0, word1, isValid);
}
public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
@ -402,15 +214,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
public void clearAndFlushDictionary() {
// Clear the node structure on memory
clearDictionary();
mBigramListLock.lock();
try {
// Clear the bigram list on memory
mBigramList.evictAll();
} finally {
mBigramListLock.unlock();
}
clear();
// Then flush the cleared state of the dictionary on disk.
flushPendingWrites();
asyncWriteBinaryDictionary();
}
}

5
java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java → java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
View File

@ -26,8 +26,8 @@ import android.util.Log;
import java.lang.ref.SoftReference;
import java.util.concurrent.ConcurrentHashMap;
public class PersonalizationDictionaryHelper {
private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName();
public class PersonalizationHelper {
private static final String TAG = PersonalizationHelper.class.getSimpleName();
private static final boolean DEBUG = false;
private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
@ -52,6 +52,7 @@ public class PersonalizationDictionaryHelper {
if (DEBUG) {
Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
}
dict.asyncReloadDictionaryIfRequired();
return dict;
}
}

8
java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
View File

@ -27,11 +27,11 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA);
super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA,
getDictionaryFileName(locale));
}
@Override
protected String getDictionaryFileName() {
return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}

1
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
View File

@ -45,6 +45,7 @@ public final class UserHistoryDictionaryBigramList {
/**
* Called when the user typed a word.
*/
@UsedForTesting
public void addBigram(String word1, String word2) {
addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE);
}

7
java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
View File

@ -31,11 +31,10 @@ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionary
UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY);
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
}
@Override
protected String getDictionaryFileName() {
return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}

3
tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java
View File

@ -28,8 +28,7 @@ public class ExpandableDictionaryTests extends AndroidTestCase {
private final static int UNIGRAM_FREQ = 50;
public void testAddWordAndGetWordFrequency() {
final ExpandableDictionary dict = new ExpandableDictionary(getContext(),
Dictionary.TYPE_USER);
final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER);
// Add words
dict.addWord("abcde", "abcde", UNIGRAM_FREQ);

2
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -86,7 +86,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final Random random) {
final List<String> words = generateWords(numberOfWords, random);
final UserHistoryPredictionDictionary dict =
PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(getContext(),
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);