Merge "Make DynamicPred...Base extend ExpandableBinaryDictionary."

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -29,6 +29,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
@@ -92,6 +93,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /* A extension for a binary dictionary file. */
     public static final String DICT_FILE_EXTENSION = ".dict";
 
+    private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask =
+            new AtomicReference<AsyncWriteBinaryDictionaryTask>();
+
     /**
      * Abstract method for loading the unigrams and bigrams of a given dictionary in a background
      * thread.
@@ -180,6 +184,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         }
     }
 
+    protected void clear() {
+        mLocalDictionaryController.writeLock().lock();
+        try {
+            mDictionaryWriter.clear();
+        } finally {
+            mLocalDictionaryController.writeLock().unlock();
+        }
+    }
+
     /**
      * Adds a word unigram to the dictionary. Used for loading a dictionary.
      */
@@ -267,7 +280,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                 final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
                         mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
                                 blockOffensiveWords);
-                if (mBinaryDictionary != null) {
+                // TODO: Remove checking mIsUpdatable and use native suggestion.
+                if (mBinaryDictionary != null && !mIsUpdatable) {
                     final ArrayList<SuggestedWordInfo> binarySuggestion =
                             mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
                                     blockOffensiveWords);
@@ -276,7 +290,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                     } else if (binarySuggestion == null) {
                         return inMemDictSuggestion;
                     } else {
-                        binarySuggestion.addAll(binarySuggestion);
+                        binarySuggestion.addAll(inMemDictSuggestion);
                         return binarySuggestion;
                     }
                 } else {
@@ -402,7 +416,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /**
      * Reloads the dictionary if required. Reload will occur asynchronously in a separate thread.
      */
-    void asyncReloadDictionaryIfRequired() {
+    public void asyncReloadDictionaryIfRequired() {
         if (!isReloadRequired()) return;
         if (DEBUG) {
             Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename);
@@ -413,7 +427,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /**
      * Reloads the dictionary if required.
      */
-    protected final void syncReloadDictionaryIfRequired() {
+    public final void syncReloadDictionaryIfRequired() {
         if (!isReloadRequired()) return;
         syncReloadDictionaryInternal();
     }
@@ -492,6 +506,68 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         }
     }
 
+    /**
+     * Load the dictionary to memory.
+     */
+    protected void asyncLoadDictionaryToMemory() {
+        new AsyncLoadDictionaryToMemoryTask().start();
+    }
+
+    /**
+     * Thread class for asynchronously loading dictionary to memory.
+     */
+    private class AsyncLoadDictionaryToMemoryTask extends Thread {
+        @Override
+        public void run() {
+            mLocalDictionaryController.writeLock().lock();
+            try {
+                mSharedDictionaryController.readLock().lock();
+                try {
+                    loadDictionaryAsync();
+                } finally {
+                    mSharedDictionaryController.readLock().unlock();
+                }
+            } finally {
+                mLocalDictionaryController.writeLock().unlock();
+            }
+        }
+    }
+
+    /**
+     * Generate binary dictionary using DictionaryWriter.
+     */
+    protected void asyncWriteBinaryDictionary() {
+        final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask();
+        newTask.start();
+        final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask);
+        if (oldTask != null) {
+            oldTask.interrupt();
+        }
+    }
+
+    /**
+     * Thread class for asynchronously writing the binary dictionary.
+     */
+    private class AsyncWriteBinaryDictionaryTask extends Thread {
+        @Override
+        public void run() {
+            mSharedDictionaryController.writeLock().lock();
+            try {
+                mLocalDictionaryController.writeLock().lock();
+                try {
+                    if (isInterrupted()) {
+                        return;
+                    }
+                    writeBinaryDictionary();
+                } finally {
+                    mLocalDictionaryController.writeLock().unlock();
+                }
+            } finally {
+                mSharedDictionaryController.writeLock().unlock();
+            }
+        }
+    }
+
     /**
      * Lock for controlling access to a given binary dictionary and for tracking whether the
      * dictionary is out of date. Can be shared across multiple dictionary instances that access the

java/src/com/android/inputmethod/latin/ExpandableDictionary.java

@@ -16,7 +16,6 @@
 
 package com.android.inputmethod.latin;
 
-import android.content.Context;
 import android.text.TextUtils;
 import android.util.Log;
 
@@ -30,9 +29,10 @@ import java.util.ArrayList;
 import java.util.LinkedList;
 
 /**
- * Base class for an in-memory dictionary that can grow dynamically and can
+ * Class for an in-memory dictionary that can grow dynamically and can
  * be searched for suggestions and valid words.
  */
+// TODO: Remove after binary dictionary supports dynamic update.
 public class ExpandableDictionary extends Dictionary {
     private static final String TAG = ExpandableDictionary.class.getSimpleName();
     /**
@@ -40,23 +40,11 @@ public class ExpandableDictionary extends Dictionary {
      */
     private static final int FULL_WORD_SCORE_MULTIPLIER = 2;
 
-    // Bigram frequency is a fixed point number with 1 meaning 1.2 and 255 meaning 1.8.
-    protected static final int BIGRAM_MAX_FREQUENCY = 255;
-
-    private Context mContext;
     private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH];
     private int mMaxDepth;
     private int mInputLength;
 
-    private boolean mRequiresReload;
-
-    private boolean mUpdatingDictionary;
-
-    // Use this lock before touching mUpdatingDictionary & mRequiresDownload
-    private Object mUpdatingLock = new Object();
-
     private static final class Node {
-        Node() {}
         char mCode;
         int mFrequency;
         boolean mTerminal;
@@ -158,46 +146,12 @@ public class ExpandableDictionary extends Dictionary {
 
     private int[][] mCodes;
 
-    public ExpandableDictionary(final Context context, final String dictType) {
+    public ExpandableDictionary(final String dictType) {
         super(dictType);
-        mContext = context;
         clearDictionary();
         mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][];
     }
 
-    public void loadDictionary() {
-        synchronized (mUpdatingLock) {
-            startDictionaryLoadingTaskLocked();
-        }
-    }
-
-    public void startDictionaryLoadingTaskLocked() {
-        if (!mUpdatingDictionary) {
-            mUpdatingDictionary = true;
-            mRequiresReload = false;
-            new LoadDictionaryTask().start();
-        }
-    }
-
-    public void setRequiresReload(final boolean reload) {
-        synchronized (mUpdatingLock) {
-            mRequiresReload = reload;
-        }
-    }
-
-    public boolean getRequiresReload() {
-        return mRequiresReload;
-    }
-
-    /** Override to load your dictionary here, on a background thread. */
-    public void loadDictionaryAsync() {
-        // empty base implementation
-    }
-
-    public Context getContext() {
-        return mContext;
-    }
-
     public int getMaxWordLength() {
         return Constants.DICTIONARY_MAX_WORD_LENGTH;
     }
@@ -257,7 +211,6 @@ public class ExpandableDictionary extends Dictionary {
     public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
             final String prevWord, final ProximityInfo proximityInfo,
             final boolean blockOffensiveWords) {
-        if (reloadDictionaryIfRequired()) return null;
         if (composer.size() > 1) {
             if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) {
                 return null;
@@ -273,17 +226,7 @@ public class ExpandableDictionary extends Dictionary {
         }
     }
 
-    // This reloads the dictionary if required, and returns whether it's currently updating its
+    private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
-    // contents or not.
-    private boolean reloadDictionaryIfRequired() {
-        synchronized (mUpdatingLock) {
-            // If we need to update, start off a background task
-            if (mRequiresReload) startDictionaryLoadingTaskLocked();
-            return mUpdatingDictionary;
-        }
-    }
-
-    protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes,
             final String prevWordForBigrams, final ProximityInfo proximityInfo) {
         final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList();
         mInputLength = codes.size();
@@ -313,11 +256,6 @@ public class ExpandableDictionary extends Dictionary {
 
     @Override
     public synchronized boolean isValidWord(final String word) {
-        synchronized (mUpdatingLock) {
-            // If we need to update, start off a background task
-            if (mRequiresReload) startDictionaryLoadingTaskLocked();
-            if (mUpdatingDictionary) return false;
-        }
         final Node node = searchNode(mRoots, word, 0, word.length());
         // If node is null, we didn't find the word, so it's not valid.
         // If node.mShortcutOnly is true, then it exists as a shortcut but not as a word,
@@ -353,7 +291,7 @@ public class ExpandableDictionary extends Dictionary {
      * Returns the word's frequency or -1 if not found
      */
     @UsedForTesting
-    protected int getWordFrequency(final String word) {
+    public int getWordFrequency(final String word) {
         // Case-sensitive search
         final Node node = searchNode(mRoots, word, 0, word.length());
         return (node == null) ? -1 : node.mFrequency;
@@ -442,7 +380,7 @@ public class ExpandableDictionary extends Dictionary {
      * @param suggestions the list in which to add suggestions
      */
     // TODO: Share this routine with the native code for BinaryDictionary
-    protected void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
+    private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word,
             final int depth, final boolean completion, final int snr, final int inputIndex,
             final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) {
         final int count = roots.mLength;
@@ -704,17 +642,6 @@ public class ExpandableDictionary extends Dictionary {
         mRoots = new NodeArray();
     }
 
-    private final class LoadDictionaryTask extends Thread {
-        LoadDictionaryTask() {}
-        @Override
-        public void run() {
-            loadDictionaryAsync();
-            synchronized (mUpdatingLock) {
-                mUpdatingDictionary = false;
-            }
-        }
-    }
-
     private static char toLowerCase(final char c) {
         char baseChar = c;
         if (c < BASE_CHARS.length) {

java/src/com/android/inputmethod/latin/LatinIME.java

@@ -74,8 +74,8 @@ import com.android.inputmethod.keyboard.MainKeyboardView;
 import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
 import com.android.inputmethod.latin.define.ProductionFlag;
 import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
-import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper;
 import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
+import com.android.inputmethod.latin.personalization.PersonalizationHelper;
 import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
 import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
 import com.android.inputmethod.latin.settings.Settings;
@@ -566,13 +566,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
 
         final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
 
-        mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper
+        mUserHistoryPredictionDictionary = PersonalizationHelper
                 .getUserHistoryPredictionDictionary(this, localeStr, prefs);
         newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
-        mPersonalizationDictionary = PersonalizationDictionaryHelper
+        mPersonalizationDictionary = PersonalizationHelper
                 .getPersonalizationDictionary(this, localeStr, prefs);
         newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
-        mPersonalizationPredictionDictionary = PersonalizationDictionaryHelper
+        mPersonalizationPredictionDictionary = PersonalizationHelper
                 .getPersonalizationPredictionDictionary(this, localeStr, prefs);
         newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary);
 

java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java

@@ -55,7 +55,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr
 
     public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
         super(context, dictType);
-        mExpandableDictionary = new ExpandableDictionary(context, dictType);
+        mExpandableDictionary = new ExpandableDictionary(dictType);
     }
 
     @Override

java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java

@@ -18,108 +18,88 @@ package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
 import android.content.SharedPreferences;
-import android.os.AsyncTask;
 import android.util.Log;
 
 import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.keyboard.ProximityInfo;
 import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.ExpandableDictionary;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
 import com.android.inputmethod.latin.LatinImeLogger;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.WordComposer;
 import com.android.inputmethod.latin.makedict.DictDecoder;
-import com.android.inputmethod.latin.makedict.DictEncoder;
-import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
 import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
-import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
 import com.android.inputmethod.latin.settings.Settings;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
-import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
-import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
-import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * This class is a base class of a dictionary for the personalized prediction language model.
  */
-public abstract class DynamicPredictionDictionaryBase extends ExpandableDictionary {
+public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
-
     private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
     public static final boolean DBG_SAVE_RESTORE = false;
     private static final boolean DBG_STRESS_TEST = false;
     private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
 
-    private static final FormatOptions VERSION3 = new FormatOptions(3,
-            true /* supportsDynamicUpdate */);
-
     /** Any pair being typed or picked */
-    private static final int FREQUENCY_FOR_TYPED = 2;
+    public static final int FREQUENCY_FOR_TYPED = 2;
-
-    /** Maximum number of pairs. Pruning will start when databases goes above this number. */
-    private static final int MAX_HISTORY_BIGRAMS = 10000;
 
     /** Locale for which this user history dictionary is storing words */
     private final String mLocale;
 
-    private final UserHistoryDictionaryBigramList mBigramList =
+    private final String mFileName;
-            new UserHistoryDictionaryBigramList();
+
-    private final ReentrantLock mBigramListLock = new ReentrantLock();
     private final SharedPreferences mPrefs;
 
     private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
             CollectionUtils.newArrayList();
 
-    private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask;
-
     // Should always be false except when we use this class for test
     @UsedForTesting boolean mIsTest = false;
 
     /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
-            final SharedPreferences sp, final String dictionaryType) {
+            final SharedPreferences sp, final String dictionaryType, final String fileName) {
-        super(context, dictionaryType);
+        super(context, locale, dictionaryType, true);
         mLocale = locale;
+        mFileName = fileName;
         mPrefs = sp;
-        mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>();
         if (mLocale != null && mLocale.length() > 1) {
-            loadDictionary();
+            asyncLoadDictionaryToMemory();
+            asyncReloadDictionaryIfRequired();
         }
     }
 
     @Override
     public void close() {
-        flushPendingWrites();
+        // Close only binary dictionary to reuse this dictionary.
-        // Don't close the database as locale changes will require it to be reopened anyway
-        // Also, the database is written to somewhat frequently, so it needs to be kept alive
-        // throughout the life of the process.
-        // mOpenHelper.close();
-        // Ignore close because we cache PersonalizationPredictionDictionary for each language.
-        // See getInstance() above.
         // super.close();
+        closeBinaryDictionary();
+        // Flush pending writes.
+        // TODO: Remove after this class become to use a dynamic binary dictionary.
+        asyncWriteBinaryDictionary();
+        Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
     }
 
     @Override
-    protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer,
+    protected boolean hasContentChanged() {
-            final String prevWord, final ProximityInfo proximityInfo) {
+        return false;
-        // Inhibit suggestions (not predictions) for user history for now. Removing this method
+    }
-        // is enough to use it through the standard ExpandableDictionary way.
+
-        return null;
+    @Override
+    protected boolean needsToReloadBeforeWriting() {
+        return false;
     }
 
     /**
      * Return whether the passed charsequence is in the dictionary.
      */
     @Override
-    public synchronized boolean isValidWord(final String word) {
+    public boolean isValidWord(final String word) {
-        // TODO: figure out what is the correct thing to do here.
+     // Words included only in the user history should be treated as not in dictionary words.
         return false;
     }
 
@@ -131,74 +111,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
      * context, as in beginning of a sentence for example.
      * The second word may not be null (a NullPointerException would be thrown).
      */
-    public int addToPersonalizationPredictionDictionary(
+    public void addToPersonalizationPredictionDictionary(
-            final String word1, final String word2, final boolean isValid) {
+            final String word0, final String word1, final boolean isValid) {
-        if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
+        if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
-                (word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
+                (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
-            return -1;
+            return;
         }
-        if (mBigramListLock.tryLock()) {
+        addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED,
-            try {
+                false /* isNotAWord */);
-                super.addWord(
-                        word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED);
-                mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED);
         // Do not insert a word as a bigram of itself
-                if (word2.equals(word1)) {
+        if (word1.equals(word0)) {
-                    return 0;
+            return;
         }
-                final int freq;
+        if (null != word0) {
-                if (null == word1) {
+            addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid);
-                    freq = FREQUENCY_FOR_TYPED;
-                } else {
-                    freq = super.setBigramAndGetFrequency(
-                            word1, word2, new ForgettingCurveParams(isValid));
         }
-                mBigramList.addBigram(word1, word2);
-                return freq;
-            } finally {
-                mBigramListLock.unlock();
-            }
-        }
-        return -1;
     }
 
-    public boolean cancelAddingUserHistory(final String word1, final String word2) {
+    public void cancelAddingUserHistory(final String word0, final String word1) {
-        if (mBigramListLock.tryLock()) {
+        removeBigramDynamically(word0, word1);
-            try {
-                if (mBigramList.removeBigram(word1, word2)) {
-                    return super.removeBigram(word1, word2);
-                }
-            } finally {
-                mBigramListLock.unlock();
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Schedules a background thread to write any pending words to the database.
-     */
-    private void flushPendingWrites() {
-        // Create a background thread to write the pending entries
-        final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask(
-                mBigramList, mLocale, this, mPrefs, getContext()).execute());
-        if (old != null) {
-            old.cancel(false);
-        }
     }
 
     @Override
-    public final void loadDictionaryAsync() {
+    protected void loadDictionaryAsync() {
-        // This must be run on non-main thread
-        mBigramListLock.lock();
-        try {
-            loadDictionaryAsyncLocked();
-        } finally {
-            mBigramListLock.unlock();
-        }
-    }
-
-    private void loadDictionaryAsyncLocked() {
         final int[] profTotalCount = { 0 };
         final String locale = getLocale();
         if (DBG_STRESS_TEST) {
@@ -210,10 +145,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
             }
         }
         final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
-        final boolean initializing = last == 0;
         final long now = System.currentTimeMillis();
-        final String fileName = getDictionaryFileName();
+        final ExpandableBinaryDictionary dictionary = this;
-        final ExpandableDictionary dictionary = this;
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
             public void setUnigram(final String word, final String shortcutTarget,
@@ -221,29 +154,25 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
                 if (DBG_SAVE_RESTORE) {
                     Log.d(TAG, "load unigram: " + word + "," + frequency);
                 }
-                dictionary.addWord(word, shortcutTarget, frequency);
+                addWord(word, shortcutTarget, frequency, false /* isNotAWord */);
                 ++profTotalCount[0];
-                addToBigramListLocked(null, word, (byte)frequency);
             }
 
             @Override
-            public void setBigram(final String word1, final String word2, final int frequency) {
+            public void setBigram(final String word0, final String word1, final int frequency) {
-                if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
+                if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
-                        && word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
+                        && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
                     if (DBG_SAVE_RESTORE) {
-                        Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency);
+                        Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
                     }
                     ++profTotalCount[0];
-                    dictionary.setBigramAndGetFrequency(
+                    addBigram(word0, word1, frequency, last);
-                            word1, word2, initializing ? new ForgettingCurveParams(true)
-                            : new ForgettingCurveParams(frequency, now, last));
                 }
-                addToBigramListLocked(word1, word2, (byte)frequency);
             }
         };
 
         // Load the dictionary from binary file
-        final File dictFile = new File(getContext().getFilesDir(), fileName);
+        final File dictFile = new File(mContext.getFilesDir(), mFileName);
         final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
                 DictDecoder.USE_BYTEARRAY);
         try {
@@ -263,131 +192,14 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
         }
     }
 
-    protected abstract String getDictionaryFileName();
-
     protected String getLocale() {
         return mLocale;
     }
 
-    private void addToBigramListLocked(String word0, String word1, byte fcValue) {
-        mBigramList.addBigram(word0, word1, fcValue);
-    }
-
-    /**
-     * Async task to write pending words to the binarydicts.
-     */
-    private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void>
-            implements BigramDictionaryInterface {
-        private final UserHistoryDictionaryBigramList mBigramList;
-        private final boolean mAddLevel0Bigrams;
-        private final String mLocale;
-        private final DynamicPredictionDictionaryBase mDynamicPredictionDictionary;
-        private final SharedPreferences mPrefs;
-        private final Context mContext;
-
-        public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites,
-                final String locale, final DynamicPredictionDictionaryBase dict,
-                final SharedPreferences prefs, final Context context) {
-            mBigramList = pendingWrites;
-            mLocale = locale;
-            mDynamicPredictionDictionary = dict;
-            mPrefs = prefs;
-            mContext = context;
-            mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS;
-        }
-
-        @Override
-        protected Void doInBackground(final Void... v) {
-            if (isCancelled()) return null;
-            if (mDynamicPredictionDictionary.mIsTest) {
-                // If mIsTest == true, wait until the lock is released.
-                mDynamicPredictionDictionary.mBigramListLock.lock();
-                try {
-                    doWriteTaskLocked();
-                } finally {
-                    mDynamicPredictionDictionary.mBigramListLock.unlock();
-                }
-            } else if (mDynamicPredictionDictionary.mBigramListLock.tryLock()) {
-                try {
-                    doWriteTaskLocked();
-                } finally {
-                    mDynamicPredictionDictionary.mBigramListLock.unlock();
-                }
-            }
-            return null;
-        }
-
-        private void doWriteTaskLocked() {
-            if (isCancelled()) return;
-            mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null);
-
-            if (DBG_STRESS_TEST) {
-                try {
-                    Log.w(TAG, "Start stress in closing: " + mLocale);
-                    Thread.sleep(15000);
-                    Log.w(TAG, "End stress in closing");
-                } catch (InterruptedException e) {
-                    Log.e(TAG, "In stress test", e);
-                }
-            }
-
-            final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
-            final String fileName =
-                    mDynamicPredictionDictionary.getDictionaryFileName();
-            final File file = new File(mContext.getFilesDir(), fileName);
-
-            final DictEncoder dictEncoder = new Ver3DictEncoder(file);
-            UserHistoryDictIOUtils.writeDictionary(dictEncoder, this, mBigramList, VERSION3);
-
-            // Save the timestamp after we finish writing the binary dictionary.
-            Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
-            if (PROFILE_SAVE_RESTORE) {
-                final long diff = System.currentTimeMillis() - now;
-                Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms.");
-            }
-        }
-
-        @Override
-        public int getFrequency(final String word1, final String word2) {
-            final int freq;
-            if (word1 == null) { // unigram
-                freq = FREQUENCY_FOR_TYPED;
-                final byte prevFc = mBigramList.getBigrams(word1).get(word2);
-            } else { // bigram
-                final NextWord nw =
-                        mDynamicPredictionDictionary.getBigramWord(word1, word2);
-                if (nw != null) {
-                    final ForgettingCurveParams fcp = nw.getFcParams();
-                    final byte prevFc = mBigramList.getBigrams(word1).get(word2);
-                    final byte fc = fcp.getFc();
-                    final boolean isValid = fcp.isValid();
-                    if (prevFc > 0 && prevFc == fc) {
-                        freq = fc & 0xFF;
-                    } else if (UserHistoryForgettingCurveUtils.
-                            needsToSave(fc, isValid, mAddLevel0Bigrams)) {
-                        freq = fc & 0xFF;
-                    } else {
-                        // Delete this entry
-                        freq = -1;
-                    }
-                } else {
-                    // Delete this entry
-                    freq = -1;
-                }
-            }
-            return freq;
-        }
-    }
-
     @UsedForTesting
     /* package for test */ void forceAddWordForTest(
-            final String word1, final String word2, final boolean isValid) {
+            final String word0, final String word1, final boolean isValid) {
-        mBigramListLock.lock();
+        addToPersonalizationPredictionDictionary(word0, word1, isValid);
-        try {
-            addToPersonalizationPredictionDictionary(word1, word2, isValid);
-        } finally {
-            mBigramListLock.unlock();
-        }
     }
 
     public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
@@ -402,15 +214,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
 
     public void clearAndFlushDictionary() {
         // Clear the node structure on memory
-        clearDictionary();
+        clear();
-        mBigramListLock.lock();
-        try {
-            // Clear the bigram list on memory
-            mBigramList.evictAll();
-        } finally {
-            mBigramListLock.unlock();
-        }
         // Then flush the cleared state of the dictionary on disk.
-        flushPendingWrites();
+        asyncWriteBinaryDictionary();
     }
 }

java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java

@@ -26,8 +26,8 @@ import android.util.Log;
 import java.lang.ref.SoftReference;
 import java.util.concurrent.ConcurrentHashMap;
 
-public class PersonalizationDictionaryHelper {
+public class PersonalizationHelper {
-    private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName();
+    private static final String TAG = PersonalizationHelper.class.getSimpleName();
     private static final boolean DEBUG = false;
 
     private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
@@ -52,6 +52,7 @@ public class PersonalizationDictionaryHelper {
                     if (DEBUG) {
                         Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
                     }
+                    dict.asyncReloadDictionaryIfRequired();
                     return dict;
                 }
             }

java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java

@@ -27,11 +27,11 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio
 
     /* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
             final SharedPreferences sp) {
-        super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA);
+        super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA,
+                getDictionaryFileName(locale));
     }
 
-    @Override
+    private static String getDictionaryFileName(final String locale) {
-    protected String getDictionaryFileName() {
+        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
-        return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
     }
 }

java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java

@@ -45,6 +45,7 @@ public final class UserHistoryDictionaryBigramList {
     /**
      * Called when the user typed a word.
      */
+    @UsedForTesting
     public void addBigram(String word1, String word2) {
         addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE);
     }

java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java

@@ -31,11 +31,10 @@ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionary
             UserHistoryPredictionDictionary.class.getSimpleName();
     /* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
             final SharedPreferences sp) {
-        super(context, locale, sp, Dictionary.TYPE_USER_HISTORY);
+        super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
     }
 
-    @Override
+    private static String getDictionaryFileName(final String locale) {
-    protected String getDictionaryFileName() {
+        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
-        return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
     }
 }

tests/src/com/android/inputmethod/latin/ExpandableDictionaryTests.java

@@ -28,8 +28,7 @@ public class ExpandableDictionaryTests extends AndroidTestCase {
     private final static int UNIGRAM_FREQ = 50;
 
     public void testAddWordAndGetWordFrequency() {
-        final ExpandableDictionary dict = new ExpandableDictionary(getContext(),
+        final ExpandableDictionary dict = new ExpandableDictionary(Dictionary.TYPE_USER);
-                Dictionary.TYPE_USER);
 
         // Add words
         dict.addWord("abcde", "abcde", UNIGRAM_FREQ);

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java

@@ -86,7 +86,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
             final Random random) {
         final List<String> words = generateWords(numberOfWords, random);
         final UserHistoryPredictionDictionary dict =
-                PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(getContext(),
+                PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
                         testFilenameSuffix /* locale */, mPrefs);
         // Add random words to the user history dictionary.
         addToDict(dict, words);