nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make DynamicPred...Base extend ExpandableBinaryDictionary. 

Bug: 6669677
Change-Id: I06afad35d3eb73510c34d10cd4116f5bcf934f7c
main
Keisuke Kuroyanagi 2013-08-26 18:50:22 +09:00
parent b2f586b9d6
commit 6e04d65932
11 changed files with 157 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -29,6 +29,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
@ -92,6 +93,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/* A extension for a binary dictionary file. */
public static final String DICT_FILE_EXTENSION = ".dict";
private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask =
new AtomicReference<AsyncWriteBinaryDictionaryTask>();
/**
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
* thread.
@ -180,6 +184,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
protected void clear() {
mLocalDictionaryController.writeLock().lock();
try {
mDictionaryWriter.clear();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
/**
* Adds a word unigram to the dictionary. Used for loading a dictionary.
*/
@ -267,7 +280,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
if (mBinaryDictionary != null) {
// TODO: Remove checking mIsUpdatable and use native suggestion.
if (mBinaryDictionary != null && !mIsUpdatable) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
@ -276,7 +290,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
} else if (binarySuggestion == null) {
return inMemDictSuggestion;
} else {
binarySuggestion.addAll(binarySuggestion);
binarySuggestion.addAll(inMemDictSuggestion);
return binarySuggestion;
}
} else {
@ -402,7 +416,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Reloads the dictionary if required. Reload will occur asynchronously in a separate thread.
*/
void asyncReloadDictionaryIfRequired() {
public void asyncReloadDictionaryIfRequired() {
if (!isReloadRequired()) return;
if (DEBUG) {
Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename);
@ -413,7 +427,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* Reloads the dictionary if required.
*/
protected final void syncReloadDictionaryIfRequired() {
public final void syncReloadDictionaryIfRequired() {
if (!isReloadRequired()) return;
syncReloadDictionaryInternal();
}
@ -492,6 +506,68 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
/**
* Load the dictionary to memory.
*/
protected void asyncLoadDictionaryToMemory() {
new AsyncLoadDictionaryToMemoryTask().start();
}
/**
* Thread class for asynchronously loading dictionary to memory.
*/
private class AsyncLoadDictionaryToMemoryTask extends Thread {
@Override
public void run() {
mLocalDictionaryController.writeLock().lock();
try {
mSharedDictionaryController.readLock().lock();
try {
loadDictionaryAsync();
} finally {
mSharedDictionaryController.readLock().unlock();
}
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
}
/**
* Generate binary dictionary using DictionaryWriter.
*/
protected void asyncWriteBinaryDictionary() {
final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask();
newTask.start();
final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask);
if (oldTask != null) {
oldTask.interrupt();
}
}
/**
* Thread class for asynchronously writing the binary dictionary.
*/
private class AsyncWriteBinaryDictionaryTask extends Thread {
@Override
public void run() {
mSharedDictionaryController.writeLock().lock();
try {
mLocalDictionaryController.writeLock().lock();
try {
if (isInterrupted()) {
return;
}
writeBinaryDictionary();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
} finally {
mSharedDictionaryController.writeLock().unlock();
}
}
}
/**
* Lock for controlling access to a given binary dictionary and for tracking whether the
* dictionary is out of date. Can be shared across multiple dictionary instances that access the

85
java/src/com/android/inputmethod/latin/ExpandableDictionary.java
View File

@ -16,7 +16,6 @@
package com.android.inputmethod.latin;
import android.content.Context;
import android.text.TextUtils;
import android.util.Log;
@ -30,9 +29,10 @@ import java.util.ArrayList;
import java.util.LinkedList;
/**
* Base class for an in-memory dictionary that can grow dynamically and can
* Class for an in-memory dictionary that can grow dynamically and can
* be searched for suggestions and valid words.
*/
// TODO: Remove after binary dictionary supports dynamic update.
public class ExpandableDictionary extends Dictionary {
private static final String TAG = ExpandableDictionary.class.getSimpleName();
/**
@ -40,23 +40,11 @@ public class ExpandableDictionary extends Dictionary {
*/
private static final int FULL_WORD_SCORE_MULTIPLIER = 2;
// Bigram frequency is a fixed point number with 1 meaning 1.2 and 255 meaning 1.8.
protected static final int BIGRAM_MAX_FREQUENCY = 255;
private Context mContext;
private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH];
private int mMaxDepth;
private int mInputLength;
private boolean mRequiresReload;
private boolean mUpdatingDictionary;
// Use this lock before touching mUpdatingDictionary & mRequiresDownload
private Object mUpdatingLock = new Object();
private static final class Node {
Node() {}
char mCode;
int mFrequency;
boolean mTerminal;
@ -158,46 +146,12 @@ public class ExpandableDictionary extends Dictionary {
private int[][] mCodes;
public ExpandableDictionary(final Context context, final String dictType) {
public ExpandableDictionary(final String dictType) {
super(dictType);
mContext = context;
clearDictionary();
mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][];
}
public void loadDictionary() {
synchronized (mUpdatingLock) {
startDictionaryLoadingTaskLocked();
}
}
public void startDictionaryLoadingTaskLocked() {
if (!mUpdatingDictionary) {
mUpdatingDictionary = true;
mRequiresReload = false;
new LoadDictionaryTask().start();
}
}
public void setRequiresReload(final boolean reload) {
synchronized (mUpdatingLock) {
mRequiresReload = reload;
}
}
public boolean getRequiresReload() {
return mRequiresReload;
}
/** Override to load your dictionary here, on a background thread. */
public void loadDictionaryAsync() {
// empty base implementation
}
public Context getContext() {
return mContext;
}
public int getMaxWordLength() {
return Constants.DICTIONARY_MAX_WORD_LENGTH;
}
@ -257,7 +211,6 @@ public class ExpandableDictionary extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo,
final boolean blockOffensiveWords) {
if (reloadDictionaryIfRequired()) return null;
if (composer.size() > 1) {
if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) {
return null;
@ -273,17 +226,7 @@ public class ExpandableDictionary extends Dictionary {
}
}
// This reloads the dictionary if required, and returns whether it's currently updating its
// contents or not.
private boolean reloadDictionaryIfRequired() {
synchronized (mUpdatingLock) {
// If we need to update, start off a background task
if (mRequiresReload) startDictionaryLoadingTaskLocked();
return mUpdatingDictionary;
}
}
protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
final String prevWordForBigrams, final ProximityInfo proximityInfo) {
final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
mInputLength = codes.size();
@ -313,11 +256,6 @@ public class ExpandableDictionary extends Dictionary {
@Override
public synchronized boolean isValidWord(final String word) {
synchronized (mUpdatingLock) {
// If we need to update, start off a background task
if (mRequiresReload) startDictionaryLoadingTaskLocked();
if (mUpdatingDictionary) return false;
}
final Node node = searchNode(mRoots, word, 0, word.length());
// If node is null, we didn't find the word, so it's not valid.
// If node.mShortcutOnly is true, then it exists as a shortcut but not as a word,
@ -353,7 +291,7 @@ public class ExpandableDictionary extends Dictionary {
* Returns the word's frequency or -1 if not found
*/
@UsedForTesting
protected int getWordFrequency(final String word) {
public int getWordFrequency(final String word) {
// Case-sensitive search
final Node node = searchNode(mRoots, word, 0, word.length());
return (node == null) ? -1 : node.mFrequency;
@ -442,7 +380,7 @@ public class ExpandableDictionary extends Dictionary {
* @param suggestions the list in which to add suggestions
*/
// TODO: Share this routine with the native code for BinaryDictionary
protected void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
final int depth, final boolean completion, final int snr, final int inputIndex,
final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) {
final int count = roots.mLength;
@ -704,17 +642,6 @@ public class ExpandableDictionary extends Dictionary {
mRoots = new NodeArray();
}
private final class LoadDictionaryTask extends Thread {
LoadDictionaryTask() {}
@Override
public void run() {
loadDictionaryAsync();
synchronized (mUpdatingLock) {
mUpdatingDictionary = false;
}
}
}
private static char toLowerCase(final char c) {
char baseChar = c;
if (c < BASE_CHARS.length) {

8
java/src/com/android/inputmethod/latin/LatinIME.java
View File

@ -74,8 +74,8 @@ import com.android.inputmethod.keyboard.MainKeyboardView;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.define.ProductionFlag;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper;
import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
import com.android.inputmethod.latin.settings.Settings;
@ -566,13 +566,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper
mUserHistoryPredictionDictionary = PersonalizationHelper
.getUserHistoryPredictionDictionary(this, localeStr, prefs);
newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
mPersonalizationDictionary = PersonalizationDictionaryHelper
mPersonalizationDictionary = PersonalizationHelper
.getPersonalizationDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
mPersonalizationPredictionDictionary = PersonalizationDictionaryHelper
mPersonalizationPredictionDictionary = PersonalizationHelper
.getPersonalizationPredictionDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary);

2
java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
View File

@ -55,7 +55,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr
public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
super(context, dictType);
mExpandableDictionary = new ExpandableDictionary(context, dictType);
mExpandableDictionary = new ExpandableDictionary(dictType);
}
@Override

297
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -18,108 +18,88 @@ package com.android.inputmethod.latin.personalization;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.AsyncTask;
import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
/**
* This class is a base class of a dictionary for the personalized prediction language model.
*/
public abstract class DynamicPredictionDictionaryBase extends ExpandableDictionary {
public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
public static final boolean DBG_SAVE_RESTORE = false;
private static final boolean DBG_STRESS_TEST = false;
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
private static final FormatOptions VERSION3 = new FormatOptions(3,
true /* supportsDynamicUpdate */);
/** Any pair being typed or picked */
private static final int FREQUENCY_FOR_TYPED = 2;
/** Maximum number of pairs. Pruning will start when databases goes above this number. */
private static final int MAX_HISTORY_BIGRAMS = 10000;
public static final int FREQUENCY_FOR_TYPED = 2;
/** Locale for which this user history dictionary is storing words */
private final String mLocale;
private final UserHistoryDictionaryBigramList mBigramList =
new UserHistoryDictionaryBigramList();
private final ReentrantLock mBigramListLock = new ReentrantLock();
private final String mFileName;
private final SharedPreferences mPrefs;
private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
CollectionUtils.newArrayList();
private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask;
// Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false;
/* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
final SharedPreferences sp, final String dictionaryType) {
super(context, dictionaryType);
final SharedPreferences sp, final String dictionaryType, final String fileName) {
super(context, locale, dictionaryType, true);
mLocale = locale;
mFileName = fileName;
mPrefs = sp;
mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>();
if (mLocale != null && mLocale.length() > 1) {
loadDictionary();
asyncLoadDictionaryToMemory();
asyncReloadDictionaryIfRequired();
}
}
@Override
public void close() {
flushPendingWrites();
// Don't close the database as locale changes will require it to be reopened anyway
// Also, the database is written to somewhat frequently, so it needs to be kept alive
// throughout the life of the process.
// mOpenHelper.close();
// Ignore close because we cache PersonalizationPredictionDictionary for each language.
// See getInstance() above.
// Close only binary dictionary to reuse this dictionary.
// super.close();
closeBinaryDictionary();
// Flush pending writes.
// TODO: Remove after this class become to use a dynamic binary dictionary.
asyncWriteBinaryDictionary();
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
}
@Override
protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo) {
// Inhibit suggestions (not predictions) for user history for now. Removing this method
// is enough to use it through the standard ExpandableDictionary way.
return null;
protected boolean hasContentChanged() {
return false;
}
@Override
protected boolean needsToReloadBeforeWriting() {
return false;
}
/**
* Return whether the passed charsequence is in the dictionary.
*/
@Override
public synchronized boolean isValidWord(final String word) {
// TODO: figure out what is the correct thing to do here.
public boolean isValidWord(final String word) {
// Words included only in the user history should be treated as not in dictionary words.
return false;
}
@ -131,74 +111,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
public int addToPersonalizationPredictionDictionary(
final String word1, final String word2, final boolean isValid) {
if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return -1;
public void addToPersonalizationPredictionDictionary(
final String word0, final String word1, final boolean isValid) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;
}
if (mBigramListLock.tryLock()) {
try {
super.addWord(
word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED);
mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED);
addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED,
false /* isNotAWord */);
// Do not insert a word as a bigram of itself
if (word2.equals(word1)) {
return 0;
if (word1.equals(word0)) {
return;
}
final int freq;
if (null == word1) {
freq = FREQUENCY_FOR_TYPED;
} else {
freq = super.setBigramAndGetFrequency(
word1, word2, new ForgettingCurveParams(isValid));
if (null != word0) {
addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid);
}
mBigramList.addBigram(word1, word2);
return freq;
} finally {
mBigramListLock.unlock();
}
}
return -1;
}
public boolean cancelAddingUserHistory(final String word1, final String word2) {
if (mBigramListLock.tryLock()) {
try {
if (mBigramList.removeBigram(word1, word2)) {
return super.removeBigram(word1, word2);
}
} finally {
mBigramListLock.unlock();
}
}
return false;
}
/**
* Schedules a background thread to write any pending words to the database.
*/
private void flushPendingWrites() {
// Create a background thread to write the pending entries
final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask(
mBigramList, mLocale, this, mPrefs, getContext()).execute());
if (old != null) {
old.cancel(false);
}
public void cancelAddingUserHistory(final String word0, final String word1) {
removeBigramDynamically(word0, word1);
}
@Override
public final void loadDictionaryAsync() {
// This must be run on non-main thread
mBigramListLock.lock();
try {
loadDictionaryAsyncLocked();
} finally {
mBigramListLock.unlock();
}
}
private void loadDictionaryAsyncLocked() {
protected void loadDictionaryAsync() {
final int[] profTotalCount = { 0 };
final String locale = getLocale();
if (DBG_STRESS_TEST) {
@ -210,10 +145,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
}
}
final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
final boolean initializing = last == 0;
final long now = System.currentTimeMillis();
final String fileName = getDictionaryFileName();
final ExpandableDictionary dictionary = this;
final ExpandableBinaryDictionary dictionary = this;
final OnAddWordListener listener = new OnAddWordListener() {
@Override
public void setUnigram(final String word, final String shortcutTarget,
@ -221,29 +154,25 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "load unigram: " + word + "," + frequency);
}
dictionary.addWord(word, shortcutTarget, frequency);
addWord(word, shortcutTarget, frequency, false /* isNotAWord */);
++profTotalCount[0];
addToBigramListLocked(null, word, (byte)frequency);
}
@Override
public void setBigram(final String word1, final String word2, final int frequency) {
if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
&& word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
public void setBigram(final String word0, final String word1, final int frequency) {
if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
&& word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency);
Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
}
++profTotalCount[0];
dictionary.setBigramAndGetFrequency(
word1, word2, initializing ? new ForgettingCurveParams(true)
: new ForgettingCurveParams(frequency, now, last));
addBigram(word0, word1, frequency, last);
}
addToBigramListLocked(word1, word2, (byte)frequency);
}
};
// Load the dictionary from binary file
final File dictFile = new File(getContext().getFilesDir(), fileName);
final File dictFile = new File(mContext.getFilesDir(), mFileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
try {
@ -263,131 +192,14 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
}
}
protected abstract String getDictionaryFileName();
protected String getLocale() {
return mLocale;
}
private void addToBigramListLocked(String word0, String word1, byte fcValue) {
mBigramList.addBigram(word0, word1, fcValue);
}
/**
* Async task to write pending words to the binarydicts.
*/
private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void>
implements BigramDictionaryInterface {
private final UserHistoryDictionaryBigramList mBigramList;
private final boolean mAddLevel0Bigrams;
private final String mLocale;
private final DynamicPredictionDictionaryBase mDynamicPredictionDictionary;
private final SharedPreferences mPrefs;
private final Context mContext;
public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites,
final String locale, final DynamicPredictionDictionaryBase dict,
final SharedPreferences prefs, final Context context) {
mBigramList = pendingWrites;
mLocale = locale;
mDynamicPredictionDictionary = dict;
mPrefs = prefs;
mContext = context;
mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS;
}
@Override
protected Void doInBackground(final Void... v) {
if (isCancelled()) return null;
if (mDynamicPredictionDictionary.mIsTest) {
// If mIsTest == true, wait until the lock is released.
mDynamicPredictionDictionary.mBigramListLock.lock();
try {
doWriteTaskLocked();
} finally {
mDynamicPredictionDictionary.mBigramListLock.unlock();
}
} else if (mDynamicPredictionDictionary.mBigramListLock.tryLock()) {
try {
doWriteTaskLocked();
} finally {
mDynamicPredictionDictionary.mBigramListLock.unlock();
}
}
return null;
}
private void doWriteTaskLocked() {
if (isCancelled()) return;
mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null);
if (DBG_STRESS_TEST) {
try {
Log.w(TAG, "Start stress in closing: " + mLocale);
Thread.sleep(15000);
Log.w(TAG, "End stress in closing");
} catch (InterruptedException e) {
Log.e(TAG, "In stress test", e);
}
}
final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
final String fileName =
mDynamicPredictionDictionary.getDictionaryFileName();
final File file = new File(mContext.getFilesDir(), fileName);
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, mBigramList, VERSION3);
// Save the timestamp after we finish writing the binary dictionary.
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms.");
}
}
@Override
public int getFrequency(final String word1, final String word2) {
final int freq;
if (word1 == null) { // unigram
freq = FREQUENCY_FOR_TYPED;
final byte prevFc = mBigramList.getBigrams(word1).get(word2);
} else { // bigram
final NextWord nw =
mDynamicPredictionDictionary.getBigramWord(word1, word2);
if (nw != null) {
final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = mBigramList.getBigrams(word1).get(word2);
final byte fc = fcp.getFc();
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
freq = fc & 0xFF;
} else if (UserHistoryForgettingCurveUtils.
needsToSave(fc, isValid, mAddLevel0Bigrams)) {
freq = fc & 0xFF;
} else {
// Delete this entry
freq = -1;
}
} else {
// Delete this entry
freq = -1;
}
}
return freq;
}
}
@UsedForTesting
/* package for test */ void forceAddWordForTest(
final String word1, final String word2, final boolean isValid) {
mBigramListLock.lock();
try {
addToPersonalizationPredictionDictionary(word1, word2, isValid);
} finally {
mBigramListLock.unlock();
}
final String word0, final String word1, final boolean isValid) {
addToPersonalizationPredictionDictionary(word0, word1, isValid);
}
public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
@ -402,15 +214,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
public void clearAndFlushDictionary() {
// Clear the node structure on memory
clearDictionary();
mBigramListLock.lock();
try {
// Clear the bigram list on memory
mBigramList.evictAll();
} finally {
mBigramListLock.unlock();
}
clear();
// Then flush the cleared state of the dictionary on disk.
flushPendingWrites();
asyncWriteBinaryDictionary();
}
}

5
java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java → java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
View File

@ -26,8 +26,8 @@ import android.util.Log;
import java.lang.ref.SoftReference;
import java.util.concurrent.ConcurrentHashMap;
public class PersonalizationDictionaryHelper {
private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName();
public class PersonalizationHelper {
private static final String TAG = PersonalizationHelper.class.getSimpleName();
private static final boolean DEBUG = false;
private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
@ -52,6 +52,7 @@ public class PersonalizationDictionaryHelper {
if (DEBUG) {
Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
}
dict.asyncReloadDictionaryIfRequired();
return dict;
}
}

8
java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
View File

@ -27,11 +27,11 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA);
super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA,
getDictionaryFileName(locale));
}
@Override
protected String getDictionaryFileName() {
return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}

1
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java
View File

@ -45,6 +45,7 @@ public final class UserHistoryDictionaryBigramList {
/**
* Called when the user typed a word.
*/
@UsedForTesting
public void addBigram(String word1, String word2) {
addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE);
}

7
java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java
View File

@ -31,11 +31,10 @@ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionary
UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY);
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
}
@Override
protected String getDictionaryFileName() {
return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
}
}

3
tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java
View File

@ -28,8 +28,7 @@ public class ExpandableDictionaryTests extends AndroidTestCase {
private final static int UNIGRAM_FREQ = 50;
public void testAddWordAndGetWordFrequency() {
final ExpandableDictionary dict = new ExpandableDictionary(getContext(),
Dictionary.TYPE_USER);
final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER);
// Add words
dict.addWord("abcde", "abcde", UNIGRAM_FREQ);

2
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -86,7 +86,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
final Random random) {
final List<String> words = generateWords(numberOfWords, random);
final UserHistoryPredictionDictionary dict =
PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(getContext(),
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);