nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix UserHistoryBigram data contention in UserHistoryDictionary

Browse source 
Bug: 6637614
Change-Id: I34d26563e59d3b09bf35b8173dac5645ccb6a39f
This commit is contained in:
satok 2012-06-11 03:25:33 +09:00
parent 2d5ba8bbde
commit 6a71f0610c
1 changed files with 225 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

413
java/src/com/android/inputmethod/latin/UserHistoryDictionary.java
View file

@ -32,6 +32,7 @@ import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingC
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.util.HashMap; import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
/** /**
* Locally gathers stats about the words user types and various other signals like auto-correction * Locally gathers stats about the words user types and various other signals like auto-correction
@ -40,6 +41,8 @@ import java.util.concurrent.ConcurrentHashMap;
public class UserHistoryDictionary extends ExpandableDictionary { public class UserHistoryDictionary extends ExpandableDictionary {
private static final String TAG = "UserHistoryDictionary"; private static final String TAG = "UserHistoryDictionary";
public static final boolean DBG_SAVE_RESTORE = false; public static final boolean DBG_SAVE_RESTORE = false;
public static final boolean DBG_STRESS_TEST = false;
public static final boolean DBG_ALWAYS_WRITE = false;
public static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; public static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
/** Any pair being typed or picked */ /** Any pair being typed or picked */
@ -82,7 +85,7 @@ public class UserHistoryDictionary extends ExpandableDictionary {
private final UserHistoryDictionaryBigramList mBigramList = private final UserHistoryDictionaryBigramList mBigramList =
new UserHistoryDictionaryBigramList(); new UserHistoryDictionaryBigramList();
private static volatile boolean sUpdatingDB = false; private final ReentrantLock mBigramListLock = new ReentrantLock();
private final SharedPreferences mPrefs; private final SharedPreferences mPrefs;
private final static HashMap<String, String> sDictProjectionMap; private final static HashMap<String, String> sDictProjectionMap;
@ -173,28 +176,38 @@ public class UserHistoryDictionary extends ExpandableDictionary {
* The second word may not be null (a NullPointerException would be thrown). * The second word may not be null (a NullPointerException would be thrown).
*/ */
public int addToUserHistory(final String word1, String word2, boolean isValid) { public int addToUserHistory(final String word1, String word2, boolean isValid) {
super.addWord(word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); if (mBigramListLock.tryLock()) {
// Do not insert a word as a bigram of itself try {
if (word2.equals(word1)) { super.addWord(
return 0; word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED);
// Do not insert a word as a bigram of itself
if (word2.equals(word1)) {
return 0;
}
final int freq;
if (null == word1) {
freq = FREQUENCY_FOR_TYPED;
} else {
freq = super.setBigramAndGetFrequency(
word1, word2, new ForgettingCurveParams(isValid));
}
mBigramList.addBigram(word1, word2);
return freq;
} finally {
mBigramListLock.unlock();
}
} }
final int freq; return -1;
if (null == word1) {
freq = FREQUENCY_FOR_TYPED;
} else {
freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams(isValid));
}
synchronized (mBigramList) {
mBigramList.addBigram(word1, word2);
}
return freq;
} }
public boolean cancelAddingUserHistory(String word1, String word2) { public boolean cancelAddingUserHistory(String word1, String word2) {
synchronized (mBigramList) { if (mBigramListLock.tryLock()) {
if (mBigramList.removeBigram(word1, word2)) { try {
return super.removeBigram(word1, word2); if (mBigramList.removeBigram(word1, word2)) {
return super.removeBigram(word1, word2);
}
} finally {
mBigramListLock.unlock();
} }
} }
return false; return false;
@ -204,70 +217,73 @@ public class UserHistoryDictionary extends ExpandableDictionary {
* Schedules a background thread to write any pending words to the database. * Schedules a background thread to write any pending words to the database.
*/ */
private void flushPendingWrites() { private void flushPendingWrites() {
synchronized (mBigramList) { if (mBigramListLock.isLocked()) {
// Nothing pending? Return
if (mBigramList.isEmpty()) return;
// Create a background thread to write the pending entries
new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this, mPrefs).execute();
}
}
/** Used for testing purpose **/
void waitUntilUpdateDBDone() {
synchronized (mBigramList) {
while (sUpdatingDB) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
}
}
return; return;
} }
// Create a background thread to write the pending entries
new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this, mPrefs).execute();
} }
@Override @Override
public void loadDictionaryAsync() { public void loadDictionaryAsync() {
synchronized(mBigramList) { // This must be run on non-main thread
final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale); mBigramListLock.lock();
final boolean initializing = last == 0; try {
final long now = System.currentTimeMillis(); loadDictionaryAsyncLocked();
// Load the words that correspond to the current input locale } finally {
final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale }); mBigramListLock.unlock();
if (null == cursor) return; }
}
private void loadDictionaryAsyncLocked() {
if (DBG_STRESS_TEST) {
try { try {
if (cursor.moveToFirst()) { Log.w(TAG, "Start stress in loading: " + mLocale);
final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); Thread.sleep(15000);
final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); Log.w(TAG, "End stress in loading");
final int fcIndex = cursor.getColumnIndex(COLUMN_FORGETTING_CURVE_VALUE); } catch (InterruptedException e) {
while (!cursor.isAfterLast()) { }
final String word1 = cursor.getString(word1Index); }
final String word2 = cursor.getString(word2Index); final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale);
final int fc = cursor.getInt(fcIndex); final boolean initializing = last == 0;
if (DBG_SAVE_RESTORE) { final long now = System.currentTimeMillis();
Log.d(TAG, "--- Load user history: " + word1 + ", " + word2 + "," // Load the words that correspond to the current input locale
+ mLocale + "," + this); final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale });
} if (null == cursor) return;
// Safeguard against adding really long words. Stack may overflow due try {
// to recursive lookup // TODO: Call SQLiteDataBase.beginTransaction / SQLiteDataBase.endTransaction
if (null == word1) { if (cursor.moveToFirst()) {
super.addWord(word2, null /* shortcut */, fc); final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1);
} else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2);
&& word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { final int fcIndex = cursor.getColumnIndex(COLUMN_FORGETTING_CURVE_VALUE);
super.setBigramAndGetFrequency( while (!cursor.isAfterLast()) {
word1, word2, initializing ? new ForgettingCurveParams(true) final String word1 = cursor.getString(word1Index);
: new ForgettingCurveParams(fc, now, last)); final String word2 = cursor.getString(word2Index);
} final int fc = cursor.getInt(fcIndex);
mBigramList.addBigram(word1, word2, (byte)fc); if (DBG_SAVE_RESTORE) {
cursor.moveToNext(); Log.d(TAG, "--- Load user history: " + word1 + ", " + word2 + ","
+ mLocale + "," + this);
} }
// Safeguard against adding really long words. Stack may overflow due
// to recursive lookup
if (null == word1) {
super.addWord(word2, null /* shortcut */, fc);
} else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH
&& word2.length() < BinaryDictionary.MAX_WORD_LENGTH) {
super.setBigramAndGetFrequency(
word1, word2, initializing ? new ForgettingCurveParams(true)
: new ForgettingCurveParams(fc, now, last));
}
mBigramList.addBigram(word1, word2, (byte)fc);
cursor.moveToNext();
} }
} finally { }
cursor.close(); } finally {
if (PROFILE_SAVE_RESTORE) { cursor.close();
final long diff = System.currentTimeMillis() - now; if (PROFILE_SAVE_RESTORE) {
Log.w(TAG, "PROF: Load User HistoryDictionary: " final long diff = System.currentTimeMillis() - now;
+ mLocale + ", " + diff + "ms."); Log.w(TAG, "PROF: Load User HistoryDictionary: "
} + mLocale + ", " + diff + "ms.");
} }
} }
} }
@ -388,146 +404,167 @@ public class UserHistoryDictionary extends ExpandableDictionary {
} }
@Override @Override
protected void onPreExecute() { protected Void doInBackground(Void... v) {
sUpdatingDB = true; SQLiteDatabase db = null;
if (mUserHistoryDictionary.mBigramListLock.tryLock()) {
try {
try {
db = mDbHelper.getWritableDatabase();
} catch (android.database.sqlite.SQLiteCantOpenDatabaseException e) {
// If we can't open the db, don't do anything. Exit through the next test
// for non-nullity of the db variable.
}
if (null == db) {
// Not much we can do. Just exit.
return null;
}
db.beginTransaction();
return doLoadTaskLocked(db);
} finally {
if (db != null) {
db.endTransaction();
}
mUserHistoryDictionary.mBigramListLock.unlock();
}
}
return null;
} }
@Override private Void doLoadTaskLocked(SQLiteDatabase db) {
protected Void doInBackground(Void... v) { if (DBG_STRESS_TEST) {
synchronized(mBigramList) {
final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
int profTotal = 0;
int profInsert = 0;
int profDelete = 0;
SQLiteDatabase db = null;
try { try {
db = mDbHelper.getWritableDatabase(); Log.w(TAG, "Start stress in closing: " + mLocale);
} catch (android.database.sqlite.SQLiteCantOpenDatabaseException e) { Thread.sleep(15000);
// If we can't open the db, don't do anything. Exit through the next test Log.w(TAG, "End stress in closing");
// for non-nullity of the db variable. } catch (InterruptedException e) {
} }
if (null == db) { }
// Not much we can do. Just exit. final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
sUpdatingDB = false; int profTotal = 0;
return null; int profInsert = 0;
} int profDelete = 0;
db.execSQL("PRAGMA foreign_keys = ON;"); db.execSQL("PRAGMA foreign_keys = ON;");
final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams; final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams;
// Write all the entries to the db // Write all the entries to the db
for (String word1 : mBigramList.keySet()) { for (String word1 : mBigramList.keySet()) {
final HashMap<String, Byte> word1Bigrams = mBigramList.getBigrams(word1); final HashMap<String, Byte> word1Bigrams = mBigramList.getBigrams(word1);
for (String word2 : word1Bigrams.keySet()) { for (String word2 : word1Bigrams.keySet()) {
if (PROFILE_SAVE_RESTORE) { if (PROFILE_SAVE_RESTORE) {
++profTotal; ++profTotal;
}
// Get new frequency. Do not insert unigrams/bigrams which freq is "-1".
final int freq; // -1, or 0~255
if (word1 == null) { // unigram
freq = FREQUENCY_FOR_TYPED;
final byte prevFc = word1Bigrams.get(word2);
if (prevFc == FREQUENCY_FOR_TYPED) {
// No need to update since we found no changes for this entry.
// Just skip to the next entry.
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "Skip update user history: " + word1 + "," + word2
+ "," + prevFc);
}
if (!DBG_ALWAYS_WRITE) {
continue;
}
} }
// Get new frequency. Do not insert unigrams/bigrams which freq is "-1". } else { // bigram
final int freq; // -1, or 0~255 final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2);
if (word1 == null) { // unigram if (nw != null) {
freq = FREQUENCY_FOR_TYPED; final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = word1Bigrams.get(word2); final byte prevFc = word1Bigrams.get(word2);
if (prevFc == FREQUENCY_FOR_TYPED) { final byte fc = (byte)fcp.getFc();
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
// No need to update since we found no changes for this entry. // No need to update since we found no changes for this entry.
// Just skip to the next entry. // Just skip to the next entry.
if (DBG_SAVE_RESTORE) { if (DBG_SAVE_RESTORE) {
Log.d(TAG, "Skip update user history: " + word1 + "," + word2 Log.d(TAG, "Skip update user history: " + word1 + ","
+ "," + prevFc); + word2 + "," + prevFc);
} }
continue; if (!DBG_ALWAYS_WRITE) {
}
} else { // bigram
final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2);
if (nw != null) {
final ForgettingCurveParams fcp = nw.getFcParams();
final byte prevFc = word1Bigrams.get(word2);
final byte fc = (byte)fcp.getFc();
final boolean isValid = fcp.isValid();
if (prevFc > 0 && prevFc == fc) {
// No need to update since we found no changes for this entry.
// Just skip to the next entry.
if (DBG_SAVE_RESTORE) {
Log.d(TAG, "Skip update user history: " + word1 + ","
+ word2 + "," + prevFc);
}
continue; continue;
} else if (UserHistoryForgettingCurveUtils.
needsToSave(fc, isValid, addLevel0Bigram)) {
freq = fc;
} else { } else {
freq = -1; freq = fc;
} }
} else if (UserHistoryForgettingCurveUtils.
needsToSave(fc, isValid, addLevel0Bigram)) {
freq = fc;
} else { } else {
freq = -1; freq = -1;
} }
} else {
freq = -1;
}
}
// TODO: this process of making a text search for each pair each time
// is terribly inefficient. Optimize this.
// Find pair id
Cursor c = null;
try {
if (null != word1) {
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND "
+ MAIN_COLUMN_LOCALE + "=?",
new String[] { word1, word2, mLocale }, null, null,
null);
} else {
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2
+ "=? AND " + MAIN_COLUMN_LOCALE + "=?",
new String[] { word2, mLocale }, null, null, null);
} }
// TODO: this process of making a text search for each pair each time
// is terribly inefficient. Optimize this.
// Find pair id
Cursor c = null;
try {
if (null != word1) {
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND "
+ MAIN_COLUMN_LOCALE + "=?",
new String[] { word1, word2, mLocale }, null, null,
null);
} else {
c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID },
MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2
+ "=? AND " + MAIN_COLUMN_LOCALE + "=?",
new String[] { word2, mLocale }, null, null, null);
}
final int pairId; final int pairId;
if (c.moveToFirst()) { if (c.moveToFirst()) {
if (PROFILE_SAVE_RESTORE) { if (PROFILE_SAVE_RESTORE) {
++profDelete; ++profDelete;
}
// Delete existing pair
pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID));
db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?",
new String[] { Integer.toString(pairId) });
} else {
// Create new pair
Long pairIdLong = db.insert(MAIN_TABLE_NAME, null,
getContentValues(word1, word2, mLocale));
pairId = pairIdLong.intValue();
} }
if (freq > 0) { // Delete existing pair
if (PROFILE_SAVE_RESTORE) { pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID));
++profInsert; db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?",
} new String[] { Integer.toString(pairId) });
if (DBG_SAVE_RESTORE) { } else {
Log.d(TAG, "--- Save user history: " + word1 + ", " + word2 // Create new pair
+ mLocale + "," + this); Long pairIdLong = db.insert(MAIN_TABLE_NAME, null,
} getContentValues(word1, word2, mLocale));
// Insert new frequency pairId = pairIdLong.intValue();
db.insert(FREQ_TABLE_NAME, null, }
getFrequencyContentValues(pairId, freq)); if (freq > 0) {
// Update an existing bigram entry in mBigramList too in order to if (PROFILE_SAVE_RESTORE) {
// synchronize the SQL DB and mBigramList. ++profInsert;
mBigramList.updateBigram(word1, word2, (byte)freq);
} }
} finally { if (DBG_SAVE_RESTORE) {
if (c != null) { Log.d(TAG, "--- Save user history: " + word1 + ", " + word2
c.close(); + mLocale + "," + this);
} }
// Insert new frequency
db.insert(FREQ_TABLE_NAME, null,
getFrequencyContentValues(pairId, freq));
// Update an existing bigram entry in mBigramList too in order to
// synchronize the SQL DB and mBigramList.
mBigramList.updateBigram(word1, word2, (byte)freq);
}
} finally {
if (c != null) {
c.close();
} }
} }
} }
}
checkPruneData(db); checkPruneData(db);
// Save the timestamp after we finish writing the SQL DB. // Save the timestamp after we finish writing the SQL DB.
SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale); SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale);
sUpdatingDB = false; if (PROFILE_SAVE_RESTORE) {
if (PROFILE_SAVE_RESTORE) { final long diff = System.currentTimeMillis() - now;
final long diff = System.currentTimeMillis() - now; Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", "+ diff
Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", "+ diff + "ms. Total: " + profTotal + ". Insert: " + profInsert + ". Delete: "
+ "ms. Total: " + profTotal + ". Insert: " + profInsert + ". Delete: " + profDelete);
+ profDelete); }
} db.setTransactionSuccessful();
return null; return null;
} // synchronized
} }
private static ContentValues getContentValues(String word1, String word2, String locale) { private static ContentValues getContentValues(String word1, String word2, String locale) {