Merge "Add version check to open binary dictionary."

java/src/com/android/inputmethod/latin/BinaryDictionary.java

@@ -125,6 +125,7 @@ public final class BinaryDictionary extends Dictionary {
     private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
     private static native void flushWithGCNative(long dict, String filePath);
     private static native void closeNative(long dict);
+    private static native int getFormatVersionNative(long dict);
     private static native int getProbabilityNative(long dict, int[] word);
     private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
     private static native int getSuggestionsNative(long dict, long proximityInfo,
@@ -241,6 +242,10 @@ public final class BinaryDictionary extends Dictionary {
         return mNativeDict != 0;
     }
 
+    public int getFormatVersion() {
+        return getFormatVersionNative(mNativeDict);
+    }
+
     public static float calcNormalizedScore(final String before, final String after,
             final int score) {
         return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -28,6 +28,7 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
 import com.android.inputmethod.latin.utils.AsyncResultHolder;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
+import com.android.inputmethod.latin.utils.StringUtils;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -128,6 +129,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
      */
     protected abstract boolean hasContentChanged();
 
+    protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
+        return true;
+    }
+
+    protected String getFileNameExtentionToOpenDict() {
+        return "";
+    }
+
     /**
      * Gets the dictionary update controller for the given filename.
      */
@@ -238,12 +247,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             public void run() {
                 if (mDictionaryWriter == null) {
                     mBinaryDictionary.close();
-                    final File file = new File(mContext.getFilesDir(), mFilename + "/"
+                    final File file = new File(mContext.getFilesDir(), mFilename);
-                            + FormatSpec.TRIE_FILE_EXTENSION);
+                    file.delete();
                     BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
                             DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
+                    // We have 'fileToOpen' in addition to 'file' for the v4 dictionary format
+                    // where 'file' is a directory, and 'fileToOpen' is a normal file.
+                    final File fileToOpen = new File(mContext.getFilesDir(), mFilename
+                            + getFileNameExtentionToOpenDict());
+                    // TODO: Make BinaryDictionary's constructor be able to accept filename
+                    // without extension.
                     mBinaryDictionary = new BinaryDictionary(
-                            file.getAbsolutePath(), 0 /* offset */, file.length(),
+                            fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(),
                             true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
                 } else {
                     mDictionaryWriter.clear();
@@ -482,8 +497,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                     + mFilenameDictionaryUpdateController.mLastUpdateTime);
         }
 
-        final File file = new File(mContext.getFilesDir(), mFilename + "/"
+        final File file = new File(mContext.getFilesDir(), mFilename
-                + FormatSpec.TRIE_FILE_EXTENSION);
+                + getFileNameExtentionToOpenDict());
         final String filename = file.getAbsolutePath();
         final long length = file.length();
 
@@ -526,8 +541,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             loadDictionaryAsync();
             mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
         } else {
-            if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
+            if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()
+                    || !isValidBinaryDictFormatVersion(mBinaryDictionary.getFormatVersion())) {
                 final File file = new File(mContext.getFilesDir(), mFilename);
+                file.delete();
                 BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
                         DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
             } else {
@@ -623,8 +640,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                         // load the shared dictionary.
                         loadBinaryDictionary();
                     }
-                    if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
+                    if (mBinaryDictionary != null && !(mBinaryDictionary.isValidDictionary()
-                        // Binary dictionary is not valid. Regenerate the dictionary file.
+                            && isValidBinaryDictFormatVersion(
+                                    mBinaryDictionary.getFormatVersion()))) {
+                        // Binary dictionary or its format version is not valid. Regenerate the
+                        // dictionary file.
                         mFilenameDictionaryUpdateController.mLastUpdateTime = time;
                         writeBinaryDictionary();
                         loadBinaryDictionary();

java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java

@@ -56,6 +56,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
     public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
     public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
 
+    public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
+
     /** Locale for which this user history dictionary is storing words */
     private final String mLocale;
 
@@ -114,6 +116,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
         return false;
     }
 
+    @Override
+    protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
+        return formatVersion >= REQUIRED_BINARY_DICTIONARY_VERSION;
+    }
+
+    @Override
+    protected String getFileNameExtentionToOpenDict() {
+        return "/" + FormatSpec.TRIE_FILE_EXTENSION;
+    }
+
     public void addMultipleDictionaryEntriesToDictionary(
             final ArrayList<LanguageModelParam> languageModelParams,
             final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

@@ -135,6 +135,12 @@ static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dic
     delete dictionary;
 }
 
+static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) {
+    Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+    if (!dictionary) return 0;
+    return dictionary->getFormatVersionNumber();
+}
+
 static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict,
         jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray,
         jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray,
@@ -428,6 +434,11 @@ static const JNINativeMethod sMethods[] = {
         const_cast<char *>("(J)V"),
         reinterpret_cast<void *>(latinime_BinaryDictionary_close)
     },
+    {
+        const_cast<char *>("getFormatVersionNative"),
+        const_cast<char *>("(J)I"),
+        reinterpret_cast<void *>(latinime_BinaryDictionary_getFormatVersion)
+    },
     {
         const_cast<char *>("flushNative"),
         const_cast<char *>("(JLjava/lang/String;)V"),

native/jni/src/suggest/core/dictionary/dictionary.h

@@ -22,6 +22,7 @@
 #include "defines.h"
 #include "jni.h"
 #include "suggest/core/dictionary/bigram_dictionary.h"
+#include "suggest/core/policy/dictionary_header_structure_policy.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/core/suggest_interface.h"
 #include "utils/exclusive_ownership_pointer.h"
@@ -92,6 +93,11 @@ class Dictionary {
         return mDictionaryStructureWithBufferPolicy.get();
     }
 
+    int getFormatVersionNumber() const {
+        return mDictionaryStructureWithBufferPolicy.get()->getHeaderStructurePolicy()
+                ->getFormatVersionNumber();
+    }
+
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
 

native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h

@@ -29,6 +29,8 @@ class DictionaryHeaderStructurePolicy {
  public:
     virtual ~DictionaryHeaderStructurePolicy() {}
 
+    virtual int getFormatVersionNumber() const = 0;
+
     virtual bool supportsDynamicUpdate() const = 0;
 
     virtual bool requiresGermanUmlautProcessing() const = 0;

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

@@ -66,6 +66,19 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
 
     ~HeaderPolicy() {}
 
+    virtual int getFormatVersionNumber() const {
+        switch (mDictFormatVersion) {
+            case FormatUtils::VERSION_2:
+                return 2;
+            case FormatUtils::VERSION_3:
+                return 3;
+            case FormatUtils::VERSION_4:
+                return 4;
+            default:
+                return 0;
+        }
+    }
+
     AK_FORCE_INLINE int getSize() const {
         return mSize;
     }

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp

@@ -34,6 +34,13 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
     FileUtils::getFilePathWithSuffix(dictDirPath,
             DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize,
             tmpDirPath);
+    if (FileUtils::existsDir(tmpDirPath)) {
+        if (!FileUtils::removeDirAndFiles(tmpDirPath)) {
+            AKLOGE("Existing directory %s cannot be removed.", tmpDirPath);
+            ASSERT(false);
+            return false;
+        }
+    }
     if (mkdir(tmpDirPath, S_IRWXU) == -1) {
         AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
         return false;

native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp

@@ -41,6 +41,15 @@ namespace latinime {
     return static_cast<int>(statBuf.st_size);
 }
 
+/* static */ bool FileUtils::existsDir(const char *const dirPath) {
+    DIR *const dir = opendir(dirPath);
+    if (dir == NULL) {
+        return false;
+    }
+    closedir(dir);
+    return true;
+}
+
 // Remove a directory and all files in the directory.
 /* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
     DIR *const dir = opendir(dirPath);
@@ -58,9 +67,11 @@ namespace latinime {
         getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath);
         if (remove(filePath) != 0) {
             AKLOGE("Cannot remove file %s.", filePath);
+            closedir(dir);
             return false;
         }
     }
+    closedir(dir);
     if (remove(dirPath) != 0) {
         AKLOGE("Cannot remove directory %s.", dirPath);
         return false;

native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h

@@ -26,6 +26,8 @@ class FileUtils {
     // Returns -1 on error.
     static int getFileSize(const char *const filePath);
 
+    static bool existsDir(const char *const dirPath);
+
     // Remove a directory and all files in the directory.
     static bool removeDirAndFiles(const char *const dirPath);