nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Create ver4 dictionary files with the dictionary name."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2013-12-11 10:42:15 +00:00 committed by Android (Google) Code Review
commit 549ae3d84e
22 changed files with 180 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
View file

@ -72,7 +72,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
private final boolean mUseFirstLastBigrams;
public ContactsBinaryDictionary(final Context context, final Locale locale) {
super(context, getFilenameWithLocale(NAME, locale), locale,
super(context, getDictNameWithLocale(NAME, locale), locale,
Dictionary.TYPE_CONTACTS, false /* isUpdatable */);
mLocale = locale;
mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);

174
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View file

@ -69,14 +69,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/**
* A static map of update controllers, each of which records the time of accesses to a single
* binary dictionary file and tracks whether the file is regenerating. The key for this map is
* the filename and the value is the shared dictionary time recorder associated with that
* filename.
* the dictionary name and the value is the shared dictionary time recorder associated with
* that dictionary name.
*/
private static final ConcurrentHashMap<String, DictionaryUpdateController>
sFilenameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap();
sDictNameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap();
private static final ConcurrentHashMap<String, PrioritizedSerialExecutor>
sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap();
sDictNameExecutorMap = CollectionUtils.newConcurrentHashMap();
/** The application context. */
protected final Context mContext;
@ -92,11 +92,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
protected AbstractDictionaryWriter mDictionaryWriter;
/**
* The name of this dictionary, used as the filename for storing the binary dictionary. Multiple
* dictionary instances with the same filename is supported, with access controlled by
* DictionaryTimeRecorder.
* The name of this dictionary, used as a part of the filename for storing the binary
* dictionary. Multiple dictionary instances with the same name is supported, with access
* controlled by DictionaryUpdateController.
*/
private final String mFilename;
private final String mDictName;
/** Dictionary locale */
private final Locale mLocale;
@ -106,7 +106,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// TODO: remove, once dynamic operations is serialized
/** Controls updating the shared binary dictionary file across multiple instances. */
private final DictionaryUpdateController mFilenameDictionaryUpdateController;
private final DictionaryUpdateController mDictNameDictionaryUpdateController;
// TODO: remove, once dynamic operations is serialized
/** Controls updating the local binary dictionary for this instance. */
@ -144,34 +144,46 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return mBinaryDictionary.isValidDictionary();
}
protected String getFileNameExtensionToOpenDict() {
return "";
protected String getFileNameToCreateDict(final String dictName) {
return dictName + DICT_FILE_EXTENSION;
}
protected String getFileNameToOpenDict(final String dictName) {
return getFileNameToCreateDict(dictName);
}
private File getFileToCreateDict() {
return new File(mContext.getFilesDir(), getFileNameToCreateDict(mDictName));
}
private File getFileToOpenDict() {
return new File(mContext.getFilesDir(), getFileNameToOpenDict(mDictName));
}
/**
* Gets the dictionary update controller for the given filename.
* Gets the dictionary update controller for the given dictionary name.
*/
private static DictionaryUpdateController getDictionaryUpdateController(
String filename) {
DictionaryUpdateController recorder = sFilenameDictionaryUpdateControllerMap.get(filename);
final String dictName) {
DictionaryUpdateController recorder = sDictNameDictionaryUpdateControllerMap.get(dictName);
if (recorder == null) {
synchronized(sFilenameDictionaryUpdateControllerMap) {
synchronized(sDictNameDictionaryUpdateControllerMap) {
recorder = new DictionaryUpdateController();
sFilenameDictionaryUpdateControllerMap.put(filename, recorder);
sDictNameDictionaryUpdateControllerMap.put(dictName, recorder);
}
}
return recorder;
}
/**
* Gets the executor for the given filename.
* Gets the executor for the given dictionary name.
*/
private static PrioritizedSerialExecutor getExecutor(final String filename) {
PrioritizedSerialExecutor executor = sFilenameExecutorMap.get(filename);
private static PrioritizedSerialExecutor getExecutor(final String dictName) {
PrioritizedSerialExecutor executor = sDictNameExecutorMap.get(dictName);
if (executor == null) {
synchronized(sFilenameExecutorMap) {
synchronized(sDictNameExecutorMap) {
executor = new PrioritizedSerialExecutor();
sFilenameExecutorMap.put(filename, executor);
sDictNameExecutorMap.put(dictName, executor);
}
}
return executor;
@ -190,28 +202,28 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
* Creates a new expandable binary dictionary.
*
* @param context The application context of the parent.
* @param filename The filename for this binary dictionary. Multiple dictionaries with the same
* filename is supported.
* @param dictName The name of the dictionary. Multiple instances with the same
* name is supported.
* @param locale the dictionary locale.
* @param dictType the dictionary type, as a human-readable string
* @param isUpdatable whether to support dynamically updating the dictionary. Please note that
* dynamic dictionary has negative effects on memory space and computation time.
*/
public ExpandableBinaryDictionary(final Context context, final String filename,
public ExpandableBinaryDictionary(final Context context, final String dictName,
final Locale locale, final String dictType, final boolean isUpdatable) {
super(dictType);
mFilename = filename;
mDictName = dictName;
mContext = context;
mLocale = locale;
mIsUpdatable = isUpdatable;
mBinaryDictionary = null;
mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename);
mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName);
// Currently, only dynamic personalization dictionary is updatable.
mDictionaryWriter = getDictionaryWriter(context, isUpdatable);
}
protected static String getFilenameWithLocale(final String name, final Locale locale) {
return name + "." + locale.toString() + DICT_FILE_EXTENSION;
protected static String getDictNameWithLocale(final String name, final Locale locale) {
return name + "." + locale.toString();
}
/**
@ -219,7 +231,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
@Override
public void close() {
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
if (mBinaryDictionary!= null) {
@ -232,7 +244,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
protected void closeBinaryDictionary() {
// Ensure that no other threads are accessing the local binary dictionary.
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
if (mBinaryDictionary != null) {
@ -245,7 +257,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
protected Map<String, String> getHeaderAttributeMap() {
HashMap<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
@ -253,19 +265,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
protected void clear() {
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
if (mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename);
final File file = getFileToCreateDict();
file.delete();
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
// We have 'fileToOpen' in addition to 'file' for the v4 dictionary format
// where 'file' is a directory, and 'fileToOpen' is a normal file.
final File fileToOpen = new File(mContext.getFilesDir(), mFilename
+ getFileNameExtensionToOpenDict());
final File fileToOpen = getFileToOpenDict();
// TODO: Make BinaryDictionary's constructor be able to accept filename
// without extension.
mBinaryDictionary = new BinaryDictionary(
@ -305,7 +316,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
* Check whether GC is needed and run GC if required.
*/
protected void runGCIfRequired(final boolean mindsBlockByGC) {
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
runGCIfRequiredInternalLocked(mindsBlockByGC);
@ -318,13 +329,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) {
if (setProcessingLargeTaskIfNot()) {
// Run GC after currently existing time sensitive operations.
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
try {
mBinaryDictionary.flushWithGC();
} finally {
mFilenameDictionaryUpdateController.mProcessingLargeTask.set(false);
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
}
}
});
@ -339,10 +350,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
final boolean isBlacklisted, final int timestamp) {
if (!mIsUpdatable) {
Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename);
Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName);
return;
}
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
@ -359,10 +370,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final int frequency, final int timestamp) {
if (!mIsUpdatable) {
Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: "
+ mFilename);
+ mDictName);
return;
}
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
@ -377,10 +388,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
protected void removeBigramDynamically(final String word0, final String word1) {
if (!mIsUpdatable) {
Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: "
+ mFilename);
+ mDictName);
return;
}
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
runGCIfRequiredInternalLocked(true /* mindsBlockByGC */);
@ -401,10 +412,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
final AddMultipleDictionaryEntriesCallback callback) {
if (!mIsUpdatable) {
Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " +
"dictionary: " + mFilename);
"dictionary: " + mDictName);
return;
}
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
final boolean locked = setProcessingLargeTaskIfNot();
@ -417,7 +428,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
callback.onFinished();
}
if (locked) {
mFilenameDictionaryUpdateController.mProcessingLargeTask.set(false);
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
}
}
}
@ -435,7 +446,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder =
new AsyncResultHolder<ArrayList<SuggestedWordInfo>>();
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
if (mBinaryDictionary == null) {
@ -471,7 +482,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return false;
}
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>();
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
holder.set(isValidWordLocked(word));
@ -505,23 +516,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
private void loadBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Loading binary dictionary: " + mFilename + " request="
+ mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ mFilenameDictionaryUpdateController.mLastUpdateTime);
Log.d(TAG, "Loading binary dictionary: " + mDictName + " request="
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ mDictNameDictionaryUpdateController.mLastUpdateTime);
}
if (DBG_STRESS_TEST) {
// Test if this class does not cause problems when it takes long time to load binary
// dictionary.
try {
Log.w(TAG, "Start stress in loading: " + mFilename);
Log.w(TAG, "Start stress in loading: " + mDictName);
Thread.sleep(15000);
Log.w(TAG, "End stress in loading");
} catch (InterruptedException e) {
}
}
final File file = new File(mContext.getFilesDir(), mFilename
+ getFileNameExtensionToOpenDict());
final File file = getFileToOpenDict();
final String filename = file.getAbsolutePath();
final long length = file.length();
@ -533,7 +543,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// swapping in the new one.
// TODO: Ensure multi-thread assignment of mBinaryDictionary.
final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
mBinaryDictionary = newBinaryDictionary;
@ -555,20 +565,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
private void writeBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Generating binary dictionary: " + mFilename + " request="
+ mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ mFilenameDictionaryUpdateController.mLastUpdateTime);
Log.d(TAG, "Generating binary dictionary: " + mDictName + " request="
+ mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update="
+ mDictNameDictionaryUpdateController.mLastUpdateTime);
}
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
mDictionaryWriter.write(getFileNameToCreateDict(mDictName), getHeaderAttributeMap());
} else {
if (mBinaryDictionary == null || !isValidDictionary()
// TODO: remove the check below
|| !matchesExpectedBinaryDictFormatVersionForThisType(
mBinaryDictionary.getFormatVersion())) {
final File file = new File(mContext.getFilesDir(), mFilename);
final File file = getFileToCreateDict();
if (!FileUtils.deleteRecursively(file)) {
Log.e(TAG, "Can't remove a file: " + file.getName());
}
@ -594,10 +604,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
protected void setRequiresReload(final boolean requiresRebuild) {
final long time = SystemClock.uptimeMillis();
mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time;
mFilenameDictionaryUpdateController.mLastUpdateRequestTime = time;
mDictNameDictionaryUpdateController.mLastUpdateRequestTime = time;
if (DEBUG) {
Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update="
+ mFilenameDictionaryUpdateController.mLastUpdateTime);
Log.d(TAG, "Reload request: " + mDictName + ": request=" + time + " update="
+ mDictNameDictionaryUpdateController.mLastUpdateTime);
}
}
@ -619,13 +629,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
private boolean processingLargeTask() {
return mFilenameDictionaryUpdateController.mProcessingLargeTask.get();
return mDictNameDictionaryUpdateController.mProcessingLargeTask.get();
}
// Returns whether the dictionary is being used for a large task. If true, we should not use
// this dictionary for latency sensitive operations.
private boolean setProcessingLargeTaskIfNot() {
return mFilenameDictionaryUpdateController.mProcessingLargeTask.compareAndSet(
return mDictNameDictionaryUpdateController.mProcessingLargeTask.compareAndSet(
false /* expect */ , true /* update */);
}
@ -636,13 +646,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private final void reloadDictionary() {
// Ensure that only one thread attempts to read or write to the shared binary dictionary
// file at the same time.
getExecutor(mFilename).execute(new Runnable() {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
try {
final long time = SystemClock.uptimeMillis();
final boolean dictionaryFileExists = dictionaryFileExists();
if (mFilenameDictionaryUpdateController.isOutOfDate()
if (mDictNameDictionaryUpdateController.isOutOfDate()
|| !dictionaryFileExists) {
// If the shared dictionary file does not exist or is out of date, the
// first instance that acquires the lock will generate a new one.
@ -651,18 +661,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// rebuild the binary dictionary. Empty dictionaries are supported (in
// the case where loadDictionaryAsync() adds nothing) in order to
// provide a uniform framework.
mFilenameDictionaryUpdateController.mLastUpdateTime = time;
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
writeBinaryDictionary();
loadBinaryDictionary();
} else {
// If not, the reload request was unnecessary so revert
// LastUpdateRequestTime to LastUpdateTime.
mFilenameDictionaryUpdateController.mLastUpdateRequestTime =
mFilenameDictionaryUpdateController.mLastUpdateTime;
mDictNameDictionaryUpdateController.mLastUpdateRequestTime =
mDictNameDictionaryUpdateController.mLastUpdateTime;
}
} else if (mBinaryDictionary == null ||
mPerInstanceDictionaryUpdateController.mLastUpdateTime
< mFilenameDictionaryUpdateController.mLastUpdateTime) {
< mDictNameDictionaryUpdateController.mLastUpdateTime) {
// Otherwise, if the local dictionary is older than the shared dictionary,
// load the shared dictionary.
loadBinaryDictionary();
@ -670,7 +680,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// If we just loaded the binary dictionary, then mBinaryDictionary is not
// up-to-date yet so it's useless to test it right away. Schedule the check
// for right after it's loaded instead.
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
if (mBinaryDictionary != null && !(isValidDictionary()
@ -680,7 +690,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// Binary dictionary or its format version is not valid. Regenerate
// the dictionary file. writeBinaryDictionary will remove the
// existing files if appropriate.
mFilenameDictionaryUpdateController.mLastUpdateTime = time;
mDictNameDictionaryUpdateController.mLastUpdateTime = time;
writeBinaryDictionary();
loadBinaryDictionary();
}
@ -688,7 +698,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
});
} finally {
mFilenameDictionaryUpdateController.mProcessingLargeTask.set(false);
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
}
}
});
@ -696,7 +706,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// TODO: cache the file's existence so that we avoid doing a disk access each time.
private boolean dictionaryFileExists() {
final File file = new File(mContext.getFilesDir(), mFilename);
final File file = getFileToOpenDict();
return file.exists();
}
@ -711,7 +721,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
};
final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask);
getExecutor(mFilename).replaceAndExecute(oldTask, newTask);
getExecutor(mDictName).replaceAndExecute(oldTask, newTask);
}
/**
@ -732,7 +742,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@UsedForTesting
public boolean isInDictionaryForTests(final String word) {
final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>();
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
if (mDictType == Dictionary.TYPE_USER_HISTORY) {
@ -745,24 +755,24 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
@UsedForTesting
public void shutdownExecutorForTests() {
getExecutor(mFilename).shutdown();
getExecutor(mDictName).shutdown();
}
@UsedForTesting
public boolean isTerminatedForTests() {
return getExecutor(mFilename).isTerminated();
return getExecutor(mDictName).isTerminated();
}
@UsedForTesting
protected void runAfterGcForDebug(final Runnable r) {
getExecutor(mFilename).executePrioritized(new Runnable() {
getExecutor(mDictName).executePrioritized(new Runnable() {
@Override
public void run() {
try {
mBinaryDictionary.flushWithGC();
r.run();
} finally {
mFilenameDictionaryUpdateController.mProcessingLargeTask.set(false);
mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false);
}
}
});

3
java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
View file

@ -18,7 +18,6 @@ package com.android.inputmethod.latin;
import android.content.ContentProviderClient;
import android.content.ContentResolver;
import android.content.ContentUris;
import android.content.Context;
import android.database.ContentObserver;
import android.database.Cursor;
@ -81,7 +80,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
public UserBinaryDictionary(final Context context, final Locale locale,
final boolean alsoUseMoreRestrictiveLocales) {
super(context, getFilenameWithLocale(NAME, locale), locale, Dictionary.TYPE_USER,
super(context, getDictNameWithLocale(NAME, locale), locale, Dictionary.TYPE_USER,
false /* isUpdatable */);
if (null == locale) throw new NullPointerException(); // Catch the error earlier
final String localeStr = locale.toString();

22
java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
View file

@ -58,13 +58,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
/** Locale for which this user history dictionary is storing words */
private final Locale mLocale;
private final String mFileName;
private final String mDictName;
/* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
final Locale locale, final String dictionaryType, final String fileName) {
super(context, fileName, locale, dictionaryType, true);
final Locale locale, final String dictionaryType, final String dictName) {
super(context, dictName, locale, dictionaryType, true);
mLocale = locale;
mFileName = fileName;
mDictName = dictName;
if (mLocale != null && mLocale.toString().length() > 1) {
reloadDictionaryIfRequired();
}
@ -88,7 +88,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName);
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
@ -113,9 +113,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
}
@Override
protected String getFileNameExtensionToOpenDict() {
// TODO: pass the directory name instead
return "/" + FormatSpec.HEADER_FILE_EXTENSION;
protected String getFileNameToCreateDict(final String dictName) {
return dictName;
}
@Override
protected String getFileNameToOpenDict(final String dictName) {
return dictName + "/" + dictName + FormatSpec.HEADER_FILE_EXTENSION;
}
public void addMultipleDictionaryEntriesToDictionary(
@ -194,7 +198,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
};
// Load the dictionary from binary file
final File dictFile = new File(mContext.getFilesDir(), mFileName);
final File dictFile = new File(mContext.getFilesDir(), mDictName);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
if (dictDecoder == null) {

7
java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
View file

@ -17,7 +17,6 @@
package com.android.inputmethod.latin.personalization;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.util.ArrayList;
@ -33,11 +32,7 @@ public class PersonalizationDictionary extends DecayingExpandableBinaryDictionar
/* package */ PersonalizationDictionary(final Context context, final Locale locale) {
super(context, locale, Dictionary.TYPE_PERSONALIZATION,
getDictionaryFileName(locale.toString()));
}
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
getDictNameWithLocale(NAME, locale));
}
public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {

8
java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
View file

@ -17,7 +17,6 @@
package com.android.inputmethod.latin.personalization;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import java.util.Locale;
@ -31,12 +30,7 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas
/* package for tests */ static final String NAME =
UserHistoryDictionary.class.getSimpleName();
/* package */ UserHistoryDictionary(final Context context, final Locale locale) {
super(context, locale, Dictionary.TYPE_USER_HISTORY,
getDictionaryFileName(locale.toString()));
}
private static String getDictionaryFileName(final String locale) {
return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale));
}
public void cancelAddingUserHistory(final String word0, final String word1) {

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
View file

@ -73,8 +73,8 @@ class BigramDictContent : public SparseTableDictContent {
bool copyBigramList(const int bigramListPos, const int toPos);
bool flushToFile(const char *const dictDirPath) const {
return flush(dictDirPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
bool flushToFile(const char *const dictBasePath) const {
return flush(dictBasePath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION);
}

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp
View file

@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId,
return writeEntry(probabilityEntry, entryPos);
}
bool ProbabilityDictContent::flushToFile(const char *const dictDirPath) const {
bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const {
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo);
for (int i = 0; i < mSize; ++i) {
@ -81,10 +81,10 @@ bool ProbabilityDictContent::flushToFile(const char *const dictDirPath) const {
return false;
}
}
return probabilityDictContentToWrite.flush(dictDirPath,
return probabilityDictContentToWrite.flush(dictBasePath,
Ver4DictConstants::FREQ_FILE_EXTENSION);
} else {
return flush(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION);
return flush(dictBasePath, Ver4DictConstants::FREQ_FILE_EXTENSION);
}
}

4
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp
View file

@ -46,8 +46,8 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const {
return addressLookupTable->get(terminalId);
}
bool ShortcutDictContent::flushToFile(const char *const dictDirPath) const {
return flush(dictDirPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
bool ShortcutDictContent::flushToFile(const char *const dictBasePath) const {
return flush(dictBasePath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION);
}

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h
View file

@ -53,7 +53,7 @@ class ShortcutDictContent : public SparseTableDictContent {
// Returns head position of shortcut list for a PtNode specified by terminalId.
int getShortcutListHeadPos(const int terminalId) const;
bool flushToFile(const char *const dictDirPath) const;
bool flushToFile(const char *const dictBasePath) const;
bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
const ShortcutDictContent *const originalShortcutDictContent);

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h
View file

@ -59,9 +59,9 @@ class SingleDictContent : public DictContent {
return &mExpandableContentBuffer;
}
bool flush(const char *const dictDirPath, const char *const contentFileName) const {
return DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName,
&mExpandableContentBuffer);
bool flush(const char *const dictBasePath, const char *const contentFileNameSuffix) const {
return DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
contentFileNameSuffix, &mExpandableContentBuffer);
}
private:

12
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp
View file

@ -18,18 +18,18 @@
namespace latinime {
bool SparseTableDictContent::flush(const char *const dictDirPath,
const char *const lookupTableFileName, const char *const addressTableFileName,
const char *const contentFileName) const {
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, lookupTableFileName,
bool SparseTableDictContent::flush(const char *const dictBasePath,
const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix,
const char *const contentFileNameSuffix) const {
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, lookupTableFileNameSuffix,
&mExpandableLookupTableBuffer)){
return false;
}
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, addressTableFileName,
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, addressTableFileNameSuffix,
&mExpandableAddressTableBuffer)) {
return false;
}
if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName,
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, contentFileNameSuffix,
&mExpandableContentBuffer)) {
return false;
}

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp
View file

@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition(
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId));
}
bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath) const {
bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) const {
// If the used buffer size is smaller than the actual buffer size, regenerate the lookup
// table and write the new table to the file.
if (getEntryPos(mSize) < getBuffer()->getTailPosition()) {
@ -63,12 +63,12 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath) con
return false;
}
}
return lookupTableToWrite.flush(dictDirPath,
return lookupTableToWrite.flush(dictBasePath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
} else {
// We can simply use this lookup table because the buffer size has not been
// changed.
return flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
return flush(dictBasePath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
}
}

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h
View file

@ -44,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent {
return mSize;
}
bool flushToFile(const char *const dictDirPath) const;
bool flushToFile(const char *const dictBasePath) const;
bool runGCTerminalIds(TerminalIdMap *const terminalIdMap);

25
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View file

@ -17,6 +17,7 @@
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include <cerrno>
#include <cstring>
#include <sys/stat.h>
#include <sys/types.h>
@ -52,34 +53,42 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false;
}
// Get dictionary base path.
const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */;
char dictName[dictNameBufSize];
FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName);
const int dictBasePathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName);
char dictBasePath[dictBasePathBufSize];
FileUtils::getFilePath(tmpDirPath, dictName, dictBasePathBufSize, dictBasePath);
// Write header file.
if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath,
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) {
AKLOGE("Dictionary header file %s/%s cannot be written.", tmpDirPath,
AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath,
Ver4DictConstants::HEADER_FILE_EXTENSION);
return false;
}
// Write trie file.
if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath,
if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath,
Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) {
AKLOGE("Dictionary trie file %s/%s cannot be written.", tmpDirPath,
AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath,
Ver4DictConstants::TRIE_FILE_EXTENSION);
return false;
}
// Write dictionary contents.
if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath)) {
if (!mTerminalPositionLookupTable.flushToFile(dictBasePath)) {
AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath);
return false;
}
if (!mProbabilityDictContent.flushToFile(tmpDirPath)) {
if (!mProbabilityDictContent.flushToFile(dictBasePath)) {
AKLOGE("Probability dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mBigramDictContent.flushToFile(tmpDirPath)) {
if (!mBigramDictContent.flushToFile(dictBasePath)) {
AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath);
return false;
}
if (!mShortcutDictContent.flushToFile(tmpDirPath)) {
if (!mShortcutDictContent.flushToFile(dictBasePath)) {
AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath);
return false;
}

8
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View file

@ -83,11 +83,11 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
return true;
}
/* static */ bool DictFileWritingUtils::flushBufferToFileInDir(const char *const dirPath,
const char *const fileName, const BufferWithExtendableBuffer *const buffer) {
const int filePathBufSize = FileUtils::getFilePathBufSize(dirPath, fileName);
/* static */ bool DictFileWritingUtils::flushBufferToFileWithSuffix(const char *const basePath,
const char *const suffix, const BufferWithExtendableBuffer *const buffer) {
const int filePathBufSize = FileUtils::getFilePathWithSuffixBufSize(basePath, suffix);
char filePath[filePathBufSize];
FileUtils::getFilePath(dirPath, fileName, filePathBufSize, filePath);
FileUtils::getFilePathWithSuffix(basePath, suffix, filePathBufSize, filePath);
return flushBufferToFile(filePath, buffer);
}

2
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View file

@ -37,7 +37,7 @@ class DictFileWritingUtils {
BufferWithExtendableBuffer *const dictHeader,
BufferWithExtendableBuffer *const dictBody);
static bool flushBufferToFileInDir(const char *const dirPath, const char *const fileName,
static bool flushBufferToFileWithSuffix(const char *const basePath, const char *const suffix,
const BufferWithExtendableBuffer *const buffer);
private:

16
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
View file

@ -20,6 +20,7 @@
#include <cstring>
#include <dirent.h>
#include <fcntl.h>
#include <libgen.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
@ -138,4 +139,19 @@ namespace latinime {
}
}
/* static */ void FileUtils::getBasename(const char *const filePath,
const int outNameBufSize, char *const outName) {
const int filePathBufSize = strlen(filePath) + 1 /* terminator */;
char filePathBuf[filePathBufSize];
snprintf(filePathBuf, filePathBufSize, "%s", filePath);
const char *const baseName = basename(filePathBuf);
const int baseNameLength = strlen(baseName);
if (baseNameLength >= outNameBufSize) {
AKLOGE("outNameBufSize is too small. dirPath: %s, outNameBufSize: %d",
filePath, outNameBufSize);
return;
}
snprintf(outName, baseNameLength + 1 /* terminator */, "%s", baseName);
}
} // namespace latinime

3
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
View file

@ -48,6 +48,9 @@ class FileUtils {
static void getDirPath(const char *const filePath, const int dirPathBufSize,
char *const outDirPath);
static void getBasename(const char *const filePath, const int outNameBufSize,
char *const outName);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(FileUtils);
};

5
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View file

@ -122,12 +122,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
return new File(file, FormatSpec.HEADER_FILE_EXTENSION);
return new File(file, headerFileName);
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ FormatSpec.HEADER_FILE_EXTENSION + " cannot be created.");
+ headerFileName + " cannot be created.");
}
}

5
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View file

@ -70,12 +70,13 @@ public class BinaryDictionaryTests extends AndroidTestCase {
Map<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
final String headerFileName = file.getName() + FormatSpec.HEADER_FILE_EXTENSION;
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
return new File(file, FormatSpec.HEADER_FILE_EXTENSION);
return new File(file, headerFileName);
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath() + " "
+ FormatSpec.HEADER_FILE_EXTENSION + " cannot be created.");
+ headerFileName + " cannot be created.");
}
}

10
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View file

@ -20,7 +20,6 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
@ -137,8 +136,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
Log.d(TAG, "This test can be used for profiling.");
Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis();
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix;
final int numberOfWords = 1000;
final Random random = new Random(123456);
@ -172,8 +170,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
// Create filename suffixes for this test.
for (int i = 0; i < numberOfLanguages; i++) {
testFilenameSuffixes[i] = "test_switching_languages" + i;
final String fileName = UserHistoryDictionary.NAME + "." +
testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i];
dictFiles[i] = new File(getContext().getFilesDir(), fileName);
clearHistory(testFilenameSuffixes[i]);
}
@ -217,8 +214,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
} finally {
Log.d(TAG, "waiting for writing ...");
waitForWriting(testFilenameSuffix);
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix
+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix;
final File dictFile = new File(getContext().getFilesDir(), fileName);
if (dictFile != null) {
assertTrue(dictFile.exists());