nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Cache UserHistoryDictionary for each language" into jb-dev

Browse source
This commit is contained in:
Satoshi Kataoka 2012-06-06 10:43:50 -07:00 committed by Android (Google) Code Review
commit 5a8d2fd1d3
3 changed files with 216 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/LatinIME.java
View file

@ -497,7 +497,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
// Note that the calling sequence of onCreate() and onCurrentInputMethodSubtypeChanged() // Note that the calling sequence of onCreate() and onCurrentInputMethodSubtypeChanged()
// is not guaranteed. It may even be called at the same time on a different thread. // is not guaranteed. It may even be called at the same time on a different thread.
if (null == mPrefs) mPrefs = PreferenceManager.getDefaultSharedPreferences(this); if (null == mPrefs) mPrefs = PreferenceManager.getDefaultSharedPreferences(this);
mUserHistoryDictionary = new UserHistoryDictionary( mUserHistoryDictionary = UserHistoryDictionary.getInstance(
this, localeStr, Suggest.DIC_USER_HISTORY, mPrefs); this, localeStr, Suggest.DIC_USER_HISTORY, mPrefs);
mSuggest.setUserHistoryDictionary(mUserHistoryDictionary); mSuggest.setUserHistoryDictionary(mUserHistoryDictionary);
} }

314
java/src/com/android/inputmethod/latin/UserHistoryDictionary.java
View file

@ -29,7 +29,9 @@ import android.util.Log;
import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
import java.lang.ref.SoftReference;
import java.util.HashMap; import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
/** /**
* Locally gathers stats about the words user types and various other signals like auto-correction * Locally gathers stats about the words user types and various other signals like auto-correction
@ -38,6 +40,7 @@ import java.util.HashMap;
public class UserHistoryDictionary extends ExpandableDictionary { public class UserHistoryDictionary extends ExpandableDictionary {
private static final String TAG = "UserHistoryDictionary"; private static final String TAG = "UserHistoryDictionary";
public static final boolean DBG_SAVE_RESTORE = false; public static final boolean DBG_SAVE_RESTORE = false;
public static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
/** Any pair being typed or picked */ /** Any pair being typed or picked */
private static final int FREQUENCY_FOR_TYPED = 2; private static final int FREQUENCY_FOR_TYPED = 2;
@ -77,13 +80,14 @@ public class UserHistoryDictionary extends ExpandableDictionary {
/** Locale for which this user history dictionary is storing words */ /** Locale for which this user history dictionary is storing words */
private final String mLocale; private final String mLocale;
private UserHistoryDictionaryBigramList mBigramList = private final UserHistoryDictionaryBigramList mBigramList =
new UserHistoryDictionaryBigramList(); new UserHistoryDictionaryBigramList();
private final Object mPendingWritesLock = new Object();
private static volatile boolean sUpdatingDB = false; private static volatile boolean sUpdatingDB = false;
private final SharedPreferences mPrefs; private final SharedPreferences mPrefs;
private final static HashMap<String, String> sDictProjectionMap; private final static HashMap<String, String> sDictProjectionMap;
private final static ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
sLangDictCache = new ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>();
static { static {
sDictProjectionMap = new HashMap<String, String>(); sDictProjectionMap = new HashMap<String, String>();
@ -107,7 +111,26 @@ public class UserHistoryDictionary extends ExpandableDictionary {
sDeleteHistoryBigrams = deleteHistoryBigram; sDeleteHistoryBigrams = deleteHistoryBigram;
} }
public UserHistoryDictionary(final Context context, final String locale, final int dicTypeId, public synchronized static UserHistoryDictionary getInstance(
final Context context, final String locale,
final int dictTypeId, final SharedPreferences sp) {
if (sLangDictCache.containsKey(locale)) {
final SoftReference<UserHistoryDictionary> ref = sLangDictCache.get(locale);
final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (PROFILE_SAVE_RESTORE) {
Log.w(TAG, "Use cached UserHistoryDictionary for " + locale);
}
return dict;
}
}
final UserHistoryDictionary dict =
new UserHistoryDictionary(context, locale, dictTypeId, sp);
sLangDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
return dict;
}
private UserHistoryDictionary(final Context context, final String locale, final int dicTypeId,
SharedPreferences sp) { SharedPreferences sp) {
super(context, dicTypeId); super(context, dicTypeId);
mLocale = locale; mLocale = locale;
@ -123,12 +146,13 @@ public class UserHistoryDictionary extends ExpandableDictionary {
@Override @Override
public void close() { public void close() {
flushPendingWrites(); flushPendingWrites();
SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale);
// Don't close the database as locale changes will require it to be reopened anyway // Don't close the database as locale changes will require it to be reopened anyway
// Also, the database is written to somewhat frequently, so it needs to be kept alive // Also, the database is written to somewhat frequently, so it needs to be kept alive
// throughout the life of the process. // throughout the life of the process.
// mOpenHelper.close(); // mOpenHelper.close();
super.close(); // Ignore close because we cache UserHistoryDictionary for each language. See getInstance()
// above.
// super.close();
} }
/** /**
@ -160,7 +184,7 @@ public class UserHistoryDictionary extends ExpandableDictionary {
} else { } else {
freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams(isValid)); freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams(isValid));
} }
synchronized (mPendingWritesLock) { synchronized (mBigramList) {
mBigramList.addBigram(word1, word2); mBigramList.addBigram(word1, word2);
} }
@ -168,7 +192,7 @@ public class UserHistoryDictionary extends ExpandableDictionary {
} }
public boolean cancelAddingUserHistory(String word1, String word2) { public boolean cancelAddingUserHistory(String word1, String word2) {
synchronized (mPendingWritesLock) { synchronized (mBigramList) {
if (mBigramList.removeBigram(word1, word2)) { if (mBigramList.removeBigram(word1, word2)) {
return super.removeBigram(word1, word2); return super.removeBigram(word1, word2);
} }
@ -180,19 +204,17 @@ public class UserHistoryDictionary extends ExpandableDictionary {
* Schedules a background thread to write any pending words to the database. * Schedules a background thread to write any pending words to the database.
*/ */
private void flushPendingWrites() { private void flushPendingWrites() {
synchronized (mPendingWritesLock) { synchronized (mBigramList) {
// Nothing pending? Return // Nothing pending? Return
if (mBigramList.isEmpty()) return; if (mBigramList.isEmpty()) return;
// Create a background thread to write the pending entries // Create a background thread to write the pending entries
new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this).execute(); new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this, mPrefs).execute();
// Create a new map for writing new entries into while the old one is written to db
mBigramList = new UserHistoryDictionaryBigramList();
} }
} }
/** Used for testing purpose **/ /** Used for testing purpose **/
void waitUntilUpdateDBDone() { void waitUntilUpdateDBDone() {
synchronized (mPendingWritesLock) { synchronized (mBigramList) {
while (sUpdatingDB) { while (sUpdatingDB) {
try { try {
Thread.sleep(100); Thread.sleep(100);
@ -205,41 +227,46 @@ public class UserHistoryDictionary extends ExpandableDictionary {
@Override @Override
public void loadDictionaryAsync() { public void loadDictionaryAsync() {
final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale); synchronized(mBigramList) {
final long now = System.currentTimeMillis(); final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale);
// Load the words that correspond to the current input locale final long now = System.currentTimeMillis();
final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale }); // Load the words that correspond to the current input locale
if (null == cursor) return; final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale });
try { if (null == cursor) return;
if (cursor.moveToFirst()) { try {
final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); if (cursor.moveToFirst()) {
final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1);
final int fcIndex = cursor.getColumnIndex(COLUMN_FORGETTING_CURVE_VALUE); final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2);
while (!cursor.isAfterLast()) { final int fcIndex = cursor.getColumnIndex(COLUMN_FORGETTING_CURVE_VALUE);
final String word1 = cursor.getString(word1Index); while (!cursor.isAfterLast()) {
final String word2 = cursor.getString(word2Index); final String word1 = cursor.getString(word1Index);
final int fc = cursor.getInt(fcIndex); final String word2 = cursor.getString(word2Index);
if (DBG_SAVE_RESTORE) { final int fc = cursor.getInt(fcIndex);
Log.d(TAG, "--- Load user history: " + word1 + ", " + word2 + "," if (DBG_SAVE_RESTORE) {
+ mLocale + "," + this); Log.d(TAG, "--- Load user history: " + word1 + ", " + word2 + ","
} + mLocale + "," + this);
// Safeguard against adding really long words. Stack may overflow due }
// to recursive lookup // Safeguard against adding really long words. Stack may overflow due
if (null == word1) { // to recursive lookup
super.addWord(word2, null /* shortcut */, fc); if (null == word1) {
} else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH super.addWord(word2, null /* shortcut */, fc);
&& word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { } else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH
super.setBigramAndGetFrequency( && word2.length() < BinaryDictionary.MAX_WORD_LENGTH) {
word1, word2, new ForgettingCurveParams(fc, now, last)); super.setBigramAndGetFrequency(
} word1, word2, new ForgettingCurveParams(fc, now, last));
synchronized(mPendingWritesLock) { }
mBigramList.addBigram(word1, word2, (byte)fc); mBigramList.addBigram(word1, word2, (byte)fc);
cursor.moveToNext();
} }
cursor.moveToNext(); }
} finally {
cursor.close();
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.w(TAG, "PROF: Load User HistoryDictionary: "
+ mLocale + ", " + diff + "ms.");
} }
} }
} finally {
cursor.close();
} }
} }
@ -317,14 +344,16 @@ public class UserHistoryDictionary extends ExpandableDictionary {
private final DatabaseHelper mDbHelper; private final DatabaseHelper mDbHelper;
private final String mLocale; private final String mLocale;
private final UserHistoryDictionary mUserHistoryDictionary; private final UserHistoryDictionary mUserHistoryDictionary;
private final SharedPreferences mPrefs;
public UpdateDbTask( public UpdateDbTask(
DatabaseHelper openHelper, UserHistoryDictionaryBigramList pendingWrites, DatabaseHelper openHelper, UserHistoryDictionaryBigramList pendingWrites,
String locale, UserHistoryDictionary dict) { String locale, UserHistoryDictionary dict, SharedPreferences prefs) {
mBigramList = pendingWrites; mBigramList = pendingWrites;
mLocale = locale; mLocale = locale;
mDbHelper = openHelper; mDbHelper = openHelper;
mUserHistoryDictionary = dict; mUserHistoryDictionary = dict;
mPrefs = prefs;
} }
/** Prune any old data if the database is getting too big. */ /** Prune any old data if the database is getting too big. */
@ -363,37 +392,39 @@ public class UserHistoryDictionary extends ExpandableDictionary {
@Override @Override
protected Void doInBackground(Void... v) { protected Void doInBackground(Void... v) {
SQLiteDatabase db = null; synchronized(mBigramList) {
try { final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
db = mDbHelper.getWritableDatabase(); int profTotal = 0;
} catch (android.database.sqlite.SQLiteCantOpenDatabaseException e) { int profInsert = 0;
// If we can't open the db, don't do anything. Exit through the next test int profDelete = 0;
// for non-nullity of the db variable. SQLiteDatabase db = null;
} try {
if (null == db) { db = mDbHelper.getWritableDatabase();
// Not much we can do. Just exit. } catch (android.database.sqlite.SQLiteCantOpenDatabaseException e) {
sUpdatingDB = false; // If we can't open the db, don't do anything. Exit through the next test
return null; // for non-nullity of the db variable.
} }
db.execSQL("PRAGMA foreign_keys = ON;"); if (null == db) {
final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams; // Not much we can do. Just exit.
sUpdatingDB = false;
return null;
}
db.execSQL("PRAGMA foreign_keys = ON;");
final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams;
// Write all the entries to the db // Write all the entries to the db
for (String word1 : mBigramList.keySet()) { for (String word1 : mBigramList.keySet()) {
final HashMap<String, Byte> word1Bigrams = mBigramList.getBigrams(word1); final HashMap<String, Byte> word1Bigrams = mBigramList.getBigrams(word1);
for (String word2 : word1Bigrams.keySet()) { for (String word2 : word1Bigrams.keySet()) {
// Get new frequency. Do not insert shortcuts/bigrams which freq is "-1". if (PROFILE_SAVE_RESTORE) {
final int freq; // -1, or 0~255 ++profTotal;
if (word1 == null) { }
freq = FREQUENCY_FOR_TYPED; // Get new frequency. Do not insert unigrams/bigrams which freq is "-1".
} else { final int freq; // -1, or 0~255
final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2); if (word1 == null) { // unigram
if (nw != null) { freq = FREQUENCY_FOR_TYPED;
final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = word1Bigrams.get(word2); final byte prevFc = word1Bigrams.get(word2);
final byte fc = (byte)fcp.getFc(); if (prevFc == FREQUENCY_FOR_TYPED) {
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
// No need to update since we found no changes for this entry. // No need to update since we found no changes for this entry.
// Just skip to the next entry. // Just skip to the next entry.
if (DBG_SAVE_RESTORE) { if (DBG_SAVE_RESTORE) {
@ -401,67 +432,100 @@ public class UserHistoryDictionary extends ExpandableDictionary {
+ "," + prevFc); + "," + prevFc);
} }
continue; continue;
} else if (UserHistoryForgettingCurveUtils. }
needsToSave(fc, isValid, addLevel0Bigram)) { } else { // bigram
freq = fc; final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2);
if (nw != null) {
final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = word1Bigrams.get(word2);
final byte fc = (byte)fcp.getFc();
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
// No need to update since we found no changes for this entry.
// Just skip to the next entry.
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "Skip update user history: " + word1 + ","
+ word2 + "," + prevFc);
}
continue;
} else if (UserHistoryForgettingCurveUtils.
needsToSave(fc, isValid, addLevel0Bigram)) {
freq = fc;
} else {
freq = -1;
}
} else { } else {
freq = -1; freq = -1;
} }
} else {
freq = -1;
} }
} // TODO: this process of making a text search for each pair each time
// TODO: this process of making a text search for each pair each time // is terribly inefficient. Optimize this.
// is terribly inefficient. Optimize this. // Find pair id
// Find pair id Cursor c = null;
Cursor c = null; try {
try { if (null != word1) {
if (null != word1) { c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND "
MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " + MAIN_COLUMN_LOCALE + "=?",
+ MAIN_COLUMN_LOCALE + "=?", new String[] { word1, word2, mLocale }, null, null,
new String[] { word1, word2, mLocale }, null, null, null);
null); } else {
} else { c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2
MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + "=? AND " + MAIN_COLUMN_LOCALE + "=?",
+ "=? AND " + MAIN_COLUMN_LOCALE + "=?", new String[] { word2, mLocale }, null, null, null);
new String[] { word2, mLocale }, null, null, null); }
}
final int pairId;
final int pairId; if (c.moveToFirst()) {
if (c.moveToFirst()) { if (PROFILE_SAVE_RESTORE) {
// Delete existing pair ++profDelete;
pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); }
db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", // Delete existing pair
new String[] { Integer.toString(pairId) }); pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID));
} else { db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?",
// Create new pair new String[] { Integer.toString(pairId) });
Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, } else {
getContentValues(word1, word2, mLocale)); // Create new pair
pairId = pairIdLong.intValue(); Long pairIdLong = db.insert(MAIN_TABLE_NAME, null,
} getContentValues(word1, word2, mLocale));
if (freq > 0) { pairId = pairIdLong.intValue();
if (DBG_SAVE_RESTORE) { }
Log.d(TAG, "--- Save user history: " + word1 + ", " + word2 if (freq > 0) {
+ mLocale + "," + this); if (PROFILE_SAVE_RESTORE) {
++profInsert;
}
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "--- Save user history: " + word1 + ", " + word2
+ mLocale + "," + this);
}
// Insert new frequency
db.insert(FREQ_TABLE_NAME, null,
getFrequencyContentValues(pairId, freq));
// Update an existing bigram entry in mBigramList too in order to
// synchronize the SQL DB and mBigramList.
mBigramList.updateBigram(word1, word2, (byte)freq);
}
} finally {
if (c != null) {
c.close();
} }
// Insert new frequency
db.insert(FREQ_TABLE_NAME, null,
getFrequencyContentValues(pairId, freq));
}
} finally {
if (c != null) {
c.close();
} }
} }
} }
}
checkPruneData(db); checkPruneData(db);
sUpdatingDB = false; // Save the timestamp after we finish writing the SQL DB.
SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale);
return null; sUpdatingDB = false;
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", "+ diff
+ "ms. Total: " + profTotal + ". Insert: " + profInsert + ". Delete: "
+ profDelete);
}
return null;
} // synchronized
} }
private static ContentValues getContentValues(String word1, String word2, String locale) { private static ContentValues getContentValues(String word1, String word2, String locale) {

27
java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java
View file

@ -39,13 +39,19 @@ public class UserHistoryDictionaryBigramList {
mBigramMap.clear(); mBigramMap.clear();
} }
/**
* Called when the user typed a word.
*/
public void addBigram(String word1, String word2) { public void addBigram(String word1, String word2) {
addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE); addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE);
} }
/**
* Called when loaded from the SQL DB.
*/
public void addBigram(String word1, String word2, byte fcValue) { public void addBigram(String word1, String word2, byte fcValue) {
if (UserHistoryDictionary.DBG_SAVE_RESTORE) { if (UserHistoryDictionary.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2); Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
} }
final HashMap<String, Byte> map; final HashMap<String, Byte> map;
if (mBigramMap.containsKey(word1)) { if (mBigramMap.containsKey(word1)) {
@ -60,6 +66,25 @@ public class UserHistoryDictionaryBigramList {
} }
} }
/**
* Called when inserted to the SQL DB.
*/
public void updateBigram(String word1, String word2, byte fcValue) {
if (UserHistoryDictionary.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
}
final HashMap<String, Byte> map;
if (mBigramMap.containsKey(word1)) {
map = mBigramMap.get(word1);
} else {
return;
}
if (!map.containsKey(word2)) {
return;
}
map.put(word2, fcValue);
}
public int size() { public int size() {
return mSize; return mSize;
} }