Merge "Set header attributes for ExpandableBinaryDictionary."

This commit is contained in:
Keisuke Kuroyanagi 2013-09-29 05:14:22 +00:00 committed by Android (Google) Code Review
commit 1cf4789ba6
14 changed files with 81 additions and 76 deletions

View file

@ -25,6 +25,7 @@ import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import java.io.File;
import java.io.IOException;
import java.util.Map;
// TODO: Quit extending Dictionary after implementing dynamic binary dictionary.
abstract public class AbstractDictionaryWriter extends Dictionary {
@ -50,16 +51,16 @@ abstract public class AbstractDictionaryWriter extends Dictionary {
abstract public void removeBigramWords(final String word0, final String word1);
abstract protected void writeDictionary(final DictEncoder dictEncoder)
throws IOException, UnsupportedFormatException;
abstract protected void writeDictionary(final DictEncoder dictEncoder,
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;
public void write(final String fileName) {
public void write(final String fileName, final Map<String, String> attributeMap) {
final String tempFileName = fileName + ".temp";
final File file = new File(mContext.getFilesDir(), fileName);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile);
writeDictionary(dictEncoder);
writeDictionary(dictEncoder, attributeMap);
tempFile.renameTo(file);
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);

View file

@ -31,6 +31,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
/**
* An in memory dictionary for memorizing entries and writing a binary dictionary.
@ -84,8 +85,11 @@ public class DictionaryWriter extends AbstractDictionaryWriter {
}
@Override
protected void writeDictionary(final DictEncoder dictEncoder)
throws IOException, UnsupportedFormatException {
protected void writeDictionary(final DictEncoder dictEncoder,
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
for (final Map.Entry<String, String> entry : attributeMap.entrySet()) {
mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue());
}
dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS);
}

View file

@ -236,6 +236,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
HashMap<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
SUPPORTS_DYNAMIC_UPDATE);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename);
return attributeMap;
}
@ -496,7 +497,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
mDictionaryWriter.write(mFilename);
mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
} else {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
@ -511,7 +512,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
} else {
mDictionaryWriter.write(mFilename);
mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
}
}
}

View file

@ -81,7 +81,7 @@ import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.settings.SettingsActivity;
import com.android.inputmethod.latin.settings.SettingsValues;
@ -179,7 +179,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
private boolean mIsMainDictionaryAvailable;
private UserBinaryDictionary mUserDictionary;
private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary;
private UserHistoryDictionary mUserHistoryDictionary;
private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary;
private PersonalizationDictionary mPersonalizationDictionary;
private boolean mIsUserDictionaryAvailable;
@ -623,9 +623,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
mUserHistoryPredictionDictionary = PersonalizationHelper
.getUserHistoryPredictionDictionary(this, localeStr, prefs);
newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary(
this, localeStr, prefs);
newSuggest.setUserHistoryDictionary(mUserHistoryDictionary);
mPersonalizationDictionary = PersonalizationHelper
.getPersonalizationDictionary(this, localeStr, prefs);
newSuggest.setPersonalizationDictionary(mPersonalizationDictionary);
@ -2750,9 +2750,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final SettingsValues currentSettings = mSettings.getCurrent();
if (!currentSettings.mCorrectionEnabled) return null;
final UserHistoryPredictionDictionary userHistoryPredictionDictionary =
mUserHistoryPredictionDictionary;
if (userHistoryPredictionDictionary == null) return null;
final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary;
if (userHistoryDictionary == null) return null;
final String prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, 2);
final String secondWord;
@ -2766,8 +2765,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
final int maxFreq = AutoCorrectionUtils.getMaxFrequency(
suggest.getUnigramDictionaries(), suggestion);
if (maxFreq == 0) return null;
userHistoryPredictionDictionary
.addToPersonalizationPredictionDictionary(prevWord, secondWord, maxFreq > 0);
userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0);
return prevWord;
}
@ -2953,7 +2951,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
}
mConnection.deleteSurroundingText(deleteLength, 0);
if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord);
mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord);
}
final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
if (mSettings.getCurrent().mCurrentLanguageHasSpaces) {

View file

@ -26,7 +26,7 @@ import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
@ -190,10 +190,8 @@ public final class Suggest {
addOrReplaceDictionaryInternal(Dictionary.TYPE_CONTACTS, contactsDictionary);
}
public void setUserHistoryPredictionDictionary(
final UserHistoryPredictionDictionary userHistoryPredictionDictionary) {
addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY,
userHistoryPredictionDictionary);
public void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) {
addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary);
}
public void setPersonalizationPredictionDictionary(

View file

@ -331,9 +331,9 @@ public final class FormatSpec {
public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
public static final String ATTRIBUTE_VALUE_TRUE = "1";
private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
public static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
public static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
public static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description";
public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
final FormatOptions formatOptions) {

View file

@ -94,6 +94,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
return attributeMap;
}
@ -117,15 +119,14 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
}
/**
* Pair will be added to the personalization prediction dictionary.
* Pair will be added to the decaying dictionary.
*
* The first word may be null. That means we don't know the context, in other words,
* it's only a unigram. The first word may also be an empty string : this means start
* context, as in beginning of a sentence for example.
* The second word may not be null (a NullPointerException would be thrown).
*/
public void addToPersonalizationPredictionDictionary(
final String word0, final String word1, final boolean isValid) {
public void addToDictionary(final String word0, final String word1, final boolean isValid) {
if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
(word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
return;

View file

@ -36,6 +36,7 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.Forge
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
// Currently this class is used to implement dynamic prodiction dictionary.
// TODO: Move to native code.
@ -113,8 +114,8 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr
}
@Override
protected void writeDictionary(final DictEncoder dictEncoder)
throws IOException, UnsupportedFormatException {
protected void writeDictionary(final DictEncoder dictEncoder,
final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
UserHistoryDictIOUtils.writeDictionary(dictEncoder,
new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
mBigramList, FORMAT_OPTIONS);

View file

@ -110,7 +110,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
if (dictionary == null) {
return;
}
dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid);
dictionary.addToDictionary(word0, word1, isValid);
}
// Bulk import
@ -122,8 +122,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
return;
}
for (final PersonalizationLanguageModelParam lmParam : lmParams) {
dictionary.addToPersonalizationPredictionDictionary(
lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
}
}
}

View file

@ -30,7 +30,7 @@ public class PersonalizationHelper {
private static final String TAG = PersonalizationHelper.class.getSimpleName();
private static final boolean DEBUG = false;
private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap();
private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>>
@ -41,25 +41,23 @@ public class PersonalizationHelper {
sLangPersonalizationPredictionDictCache =
CollectionUtils.newConcurrentHashMap();
public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary(
public static UserHistoryDictionary getUserHistoryDictionary(
final Context context, final String locale, final SharedPreferences sp) {
synchronized (sLangUserHistoryDictCache) {
if (sLangUserHistoryDictCache.containsKey(locale)) {
final SoftReference<UserHistoryPredictionDictionary> ref =
final SoftReference<UserHistoryDictionary> ref =
sLangUserHistoryDictCache.get(locale);
final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get();
final UserHistoryDictionary dict = ref == null ? null : ref.get();
if (dict != null) {
if (DEBUG) {
Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale);
Log.w(TAG, "Use cached UserHistoryDictionary for " + locale);
}
dict.reloadDictionaryIfRequired();
return dict;
}
}
final UserHistoryPredictionDictionary dict =
new UserHistoryPredictionDictionary(context, locale, sp);
sLangUserHistoryDictCache.put(
locale, new SoftReference<UserHistoryPredictionDictionary>(dict));
final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
return dict;
}
}

View file

@ -26,10 +26,10 @@ import android.content.SharedPreferences;
* Locally gathers stats about the words user types and various other signals like auto-correction
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/
public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
/* package for tests */ static final String NAME =
UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
UserHistoryDictionary.class.getSimpleName();
/* package */ UserHistoryDictionary(final Context context, final String locale,
final SharedPreferences sp) {
super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
}

View file

@ -33,6 +33,8 @@
namespace latinime {
const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32;
Dictionary::Dictionary(JNIEnv *env,
DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
@ -131,27 +133,27 @@ void Dictionary::getProperty(const char *const query, char *const outResult,
}
void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
const int BUFFER_SIZE = 16;
int dictionaryIdCodePointBuffer[BUFFER_SIZE];
int versionStringCodePointBuffer[BUFFER_SIZE];
int dateStringCodePointBuffer[BUFFER_SIZE];
int dictionaryIdCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
int versionStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
int dateStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
const DictionaryHeaderStructurePolicy *const headerPolicy =
getDictionaryStructurePolicy()->getHeaderStructurePolicy();
headerPolicy->readHeaderValueOrQuestionMark("dictionary", dictionaryIdCodePointBuffer,
BUFFER_SIZE);
HEADER_ATTRIBUTE_BUFFER_SIZE);
headerPolicy->readHeaderValueOrQuestionMark("version", versionStringCodePointBuffer,
BUFFER_SIZE);
headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer, BUFFER_SIZE);
HEADER_ATTRIBUTE_BUFFER_SIZE);
headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer,
HEADER_ATTRIBUTE_BUFFER_SIZE);
char dictionaryIdCharBuffer[BUFFER_SIZE];
char versionStringCharBuffer[BUFFER_SIZE];
char dateStringCharBuffer[BUFFER_SIZE];
intArrayToCharArray(dictionaryIdCodePointBuffer, BUFFER_SIZE,
dictionaryIdCharBuffer, BUFFER_SIZE);
intArrayToCharArray(versionStringCodePointBuffer, BUFFER_SIZE,
versionStringCharBuffer, BUFFER_SIZE);
intArrayToCharArray(dateStringCodePointBuffer, BUFFER_SIZE,
dateStringCharBuffer, BUFFER_SIZE);
char dictionaryIdCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
char versionStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
char dateStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE];
intArrayToCharArray(dictionaryIdCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
dictionaryIdCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
intArrayToCharArray(versionStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
versionStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
intArrayToCharArray(dateStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE,
dateStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE);
LogUtils::logToJava(env,
"Dictionary info: dictionary = %s ; version = %s ; date = %s",

View file

@ -95,6 +95,8 @@ class Dictionary {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
static const int HEADER_ATTRIBUTE_BUFFER_SIZE;
DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
const BigramDictionary *const mBigramDictionary;
const SuggestInterface *const mGestureSuggest;

View file

@ -75,10 +75,10 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
return new ArrayList<String>(wordSet);
}
private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) {
private void addToDict(final UserHistoryDictionary dict, final List<String> words) {
String prevWord = null;
for (String word : words) {
dict.addToPersonalizationPredictionDictionary(prevWord, word, true);
dict.addToDictionary(prevWord, word, true);
prevWord = word;
}
}
@ -90,8 +90,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords,
final Random random, final boolean checkContents) {
final List<String> words = generateWords(numberOfWords, random);
final UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
final UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix /* locale */, mPrefs);
// Add random words to the user history dictionary.
addToDict(dict, words);
@ -122,8 +122,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
true /* checksContents */);
} finally {
try {
final UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
final UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
Log.d(TAG, "waiting for writing ...");
dict.shutdownExecutorForTests();
@ -134,7 +134,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Log.d(TAG, "InterruptedException: " + e);
}
final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
@ -159,7 +159,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
testFilenameSuffixes[i] = "testSwitchingLanguages" + i;
final String fileName = UserHistoryPredictionDictionary.NAME + "." +
final String fileName = UserHistoryDictionary.NAME + "." +
testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
}
@ -181,8 +181,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
try {
Log.d(TAG, "waiting for writing ...");
for (int i = 0; i < numberOfLanguages; i++) {
final UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
final UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffixes[i], mPrefs);
dict.shutdownExecutorForTests();
while (!dict.isTerminatedForTests()) {
@ -210,8 +210,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
10000 : 1000;
final Random random = new Random(123456);
UserHistoryPredictionDictionary dict =
PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
UserHistoryDictionary dict =
PersonalizationHelper.getUserHistoryDictionary(getContext(),
testFilenameSuffix, mPrefs);
try {
addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
@ -227,7 +227,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
} catch (InterruptedException e) {
Log.d(TAG, "InterruptedException: ", e);
}
final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {