diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 51ae63631..4fa682d66 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -125,6 +125,7 @@ public final class BinaryDictionary extends Dictionary { private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); private static native void flushWithGCNative(long dict, String filePath); private static native void closeNative(long dict); + private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); private static native int getSuggestionsNative(long dict, long proximityInfo, @@ -241,6 +242,10 @@ public final class BinaryDictionary extends Dictionary { return mNativeDict != 0; } + public int getFormatVersion() { + return getFormatVersionNative(mNativeDict); + } + public static float calcNormalizedScore(final String before, final String after, final int score) { return calcNormalizedScoreNative(StringUtils.toCodePointArray(before), diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 7041df598..7107076cc 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -28,6 +28,7 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; +import com.android.inputmethod.latin.utils.StringUtils; import java.io.File; import java.util.ArrayList; @@ -128,6 +129,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected abstract boolean hasContentChanged(); + protected boolean isValidBinaryDictFormatVersion(final int formatVersion) { + return true; + } + + protected String getFileNameExtentionToOpenDict() { + return ""; + } + /** * Gets the dictionary update controller for the given filename. */ @@ -238,12 +247,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (mDictionaryWriter == null) { mBinaryDictionary.close(); - final File file = new File(mContext.getFilesDir(), mFilename + "/" - + FormatSpec.TRIE_FILE_EXTENSION); + final File file = new File(mContext.getFilesDir(), mFilename); + file.delete(); BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); + // We have 'fileToOpen' in addition to 'file' for the v4 dictionary format + // where 'file' is a directory, and 'fileToOpen' is a normal file. + final File fileToOpen = new File(mContext.getFilesDir(), mFilename + + getFileNameExtentionToOpenDict()); + // TODO: Make BinaryDictionary's constructor be able to accept filename + // without extension. mBinaryDictionary = new BinaryDictionary( - file.getAbsolutePath(), 0 /* offset */, file.length(), + fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(), true /* useFullEditDistance */, null, mDictType, mIsUpdatable); } else { mDictionaryWriter.clear(); @@ -482,8 +497,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mFilenameDictionaryUpdateController.mLastUpdateTime); } - final File file = new File(mContext.getFilesDir(), mFilename + "/" - + FormatSpec.TRIE_FILE_EXTENSION); + final File file = new File(mContext.getFilesDir(), mFilename + + getFileNameExtentionToOpenDict()); final String filename = file.getAbsolutePath(); final long length = file.length(); @@ -526,8 +541,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { loadDictionaryAsync(); mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } else { - if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { + if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary() + || !isValidBinaryDictFormatVersion(mBinaryDictionary.getFormatVersion())) { final File file = new File(mContext.getFilesDir(), mFilename); + file.delete(); BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { @@ -623,8 +640,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // load the shared dictionary. loadBinaryDictionary(); } - if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { - // Binary dictionary is not valid. Regenerate the dictionary file. + if (mBinaryDictionary != null && !(mBinaryDictionary.isValidDictionary() + && isValidBinaryDictFormatVersion( + mBinaryDictionary.getFormatVersion()))) { + // Binary dictionary or its format version is not valid. Regenerate the + // dictionary file. mFilenameDictionaryUpdateController.mLastUpdateTime = time; writeBinaryDictionary(); loadBinaryDictionary(); diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index bc1160160..8b948831e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -56,6 +56,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED; public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY; + public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4; + /** Locale for which this user history dictionary is storing words */ private final String mLocale; @@ -114,6 +116,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB return false; } + @Override + protected boolean isValidBinaryDictFormatVersion(final int formatVersion) { + return formatVersion >= REQUIRED_BINARY_DICTIONARY_VERSION; + } + + @Override + protected String getFileNameExtentionToOpenDict() { + return "/" + FormatSpec.TRIE_FILE_EXTENSION; + } + public void addMultipleDictionaryEntriesToDictionary( final ArrayList languageModelParams, final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 5fc20eaa0..fe333c7b2 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -135,6 +135,12 @@ static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dic delete dictionary; } +static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) { + Dictionary *dictionary = reinterpret_cast(dict); + if (!dictionary) return 0; + return dictionary->getFormatVersionNumber(); +} + static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict, jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray, jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray, @@ -428,6 +434,11 @@ static const JNINativeMethod sMethods[] = { const_cast("(J)V"), reinterpret_cast(latinime_BinaryDictionary_close) }, + { + const_cast("getFormatVersionNative"), + const_cast("(J)I"), + reinterpret_cast(latinime_BinaryDictionary_getFormatVersion) + }, { const_cast("flushNative"), const_cast("(JLjava/lang/String;)V"), diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 977ad35e1..d4eb47c14 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -22,6 +22,7 @@ #include "defines.h" #include "jni.h" #include "suggest/core/dictionary/bigram_dictionary.h" +#include "suggest/core/policy/dictionary_header_structure_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/core/suggest_interface.h" #include "utils/exclusive_ownership_pointer.h" @@ -92,6 +93,11 @@ class Dictionary { return mDictionaryStructureWithBufferPolicy.get(); } + int getFormatVersionNumber() const { + return mDictionaryStructureWithBufferPolicy.get()->getHeaderStructurePolicy() + ->getFormatVersionNumber(); + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index 5492c6070..b05b7c334 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -29,6 +29,8 @@ class DictionaryHeaderStructurePolicy { public: virtual ~DictionaryHeaderStructurePolicy() {} + virtual int getFormatVersionNumber() const = 0; + virtual bool supportsDynamicUpdate() const = 0; virtual bool requiresGermanUmlautProcessing() const = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 7c06a7117..b96f6aa6e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -66,6 +66,19 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { ~HeaderPolicy() {} + virtual int getFormatVersionNumber() const { + switch (mDictFormatVersion) { + case FormatUtils::VERSION_2: + return 2; + case FormatUtils::VERSION_3: + return 3; + case FormatUtils::VERSION_4: + return 4; + default: + return 0; + } + } + AK_FORCE_INLINE int getSize() const { return mSize; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index d31253153..40c9b5e78 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -34,6 +34,13 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, FileUtils::getFilePathWithSuffix(dictDirPath, DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize, tmpDirPath); + if (FileUtils::existsDir(tmpDirPath)) { + if (!FileUtils::removeDirAndFiles(tmpDirPath)) { + AKLOGE("Existing directory %s cannot be removed.", tmpDirPath); + ASSERT(false); + return false; + } + } if (mkdir(tmpDirPath, S_IRWXU) == -1) { AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp index dedcd7a99..34da76903 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp @@ -41,6 +41,15 @@ namespace latinime { return static_cast(statBuf.st_size); } +/* static */ bool FileUtils::existsDir(const char *const dirPath) { + DIR *const dir = opendir(dirPath); + if (dir == NULL) { + return false; + } + closedir(dir); + return true; +} + // Remove a directory and all files in the directory. /* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) { DIR *const dir = opendir(dirPath); @@ -58,9 +67,11 @@ namespace latinime { getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath); if (remove(filePath) != 0) { AKLOGE("Cannot remove file %s.", filePath); + closedir(dir); return false; } } + closedir(dir); if (remove(dirPath) != 0) { AKLOGE("Cannot remove directory %s.", dirPath); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h index 7dcdef85f..e55837337 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h @@ -26,6 +26,8 @@ class FileUtils { // Returns -1 on error. static int getFileSize(const char *const filePath); + static bool existsDir(const char *const dirPath); + // Remove a directory and all files in the directory. static bool removeDirAndFiles(const char *const dirPath);