Add version check to open binary dictionary.

Bug: 11073222
Change-Id: Iabae12b8f14a9da3cdc804dcc6af443cedeb3f99
main
Keisuke Kuroyanagi 2013-11-28 20:40:26 +09:00
parent c7dd2eb7ad
commit 8ac3c65b00
10 changed files with 97 additions and 8 deletions

View File

@ -125,6 +125,7 @@ public final class BinaryDictionary extends Dictionary {
private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
private static native void flushWithGCNative(long dict, String filePath); private static native void flushWithGCNative(long dict, String filePath);
private static native void closeNative(long dict); private static native void closeNative(long dict);
private static native int getFormatVersionNative(long dict);
private static native int getProbabilityNative(long dict, int[] word); private static native int getProbabilityNative(long dict, int[] word);
private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
private static native int getSuggestionsNative(long dict, long proximityInfo, private static native int getSuggestionsNative(long dict, long proximityInfo,
@ -241,6 +242,10 @@ public final class BinaryDictionary extends Dictionary {
return mNativeDict != 0; return mNativeDict != 0;
} }
public int getFormatVersion() {
return getFormatVersionNative(mNativeDict);
}
public static float calcNormalizedScore(final String before, final String after, public static float calcNormalizedScore(final String before, final String after,
final int score) { final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before), return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),

View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import com.android.inputmethod.latin.utils.StringUtils;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
@ -128,6 +129,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/ */
protected abstract boolean hasContentChanged(); protected abstract boolean hasContentChanged();
protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
return true;
}
protected String getFileNameExtentionToOpenDict() {
return "";
}
/** /**
* Gets the dictionary update controller for the given filename. * Gets the dictionary update controller for the given filename.
*/ */
@ -238,12 +247,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() { public void run() {
if (mDictionaryWriter == null) { if (mDictionaryWriter == null) {
mBinaryDictionary.close(); mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename + "/" final File file = new File(mContext.getFilesDir(), mFilename);
+ FormatSpec.TRIE_FILE_EXTENSION); file.delete();
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
// We have 'fileToOpen' in addition to 'file' for the v4 dictionary format
// where 'file' is a directory, and 'fileToOpen' is a normal file.
final File fileToOpen = new File(mContext.getFilesDir(), mFilename
+ getFileNameExtentionToOpenDict());
// TODO: Make BinaryDictionary's constructor be able to accept filename
// without extension.
mBinaryDictionary = new BinaryDictionary( mBinaryDictionary = new BinaryDictionary(
file.getAbsolutePath(), 0 /* offset */, file.length(), fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(),
true /* useFullEditDistance */, null, mDictType, mIsUpdatable); true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
} else { } else {
mDictionaryWriter.clear(); mDictionaryWriter.clear();
@ -482,8 +497,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
+ mFilenameDictionaryUpdateController.mLastUpdateTime); + mFilenameDictionaryUpdateController.mLastUpdateTime);
} }
final File file = new File(mContext.getFilesDir(), mFilename + "/" final File file = new File(mContext.getFilesDir(), mFilename
+ FormatSpec.TRIE_FILE_EXTENSION); + getFileNameExtentionToOpenDict());
final String filename = file.getAbsolutePath(); final String filename = file.getAbsolutePath();
final long length = file.length(); final long length = file.length();
@ -526,8 +541,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
loadDictionaryAsync(); loadDictionaryAsync();
mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
} else { } else {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()
|| !isValidBinaryDictFormatVersion(mBinaryDictionary.getFormatVersion())) {
final File file = new File(mContext.getFilesDir(), mFilename); final File file = new File(mContext.getFilesDir(), mFilename);
file.delete();
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else { } else {
@ -623,8 +640,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// load the shared dictionary. // load the shared dictionary.
loadBinaryDictionary(); loadBinaryDictionary();
} }
if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { if (mBinaryDictionary != null && !(mBinaryDictionary.isValidDictionary()
// Binary dictionary is not valid. Regenerate the dictionary file. && isValidBinaryDictFormatVersion(
mBinaryDictionary.getFormatVersion()))) {
// Binary dictionary or its format version is not valid. Regenerate the
// dictionary file.
mFilenameDictionaryUpdateController.mLastUpdateTime = time; mFilenameDictionaryUpdateController.mLastUpdateTime = time;
writeBinaryDictionary(); writeBinaryDictionary();
loadBinaryDictionary(); loadBinaryDictionary();

View File

@ -56,6 +56,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED; public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY; public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
/** Locale for which this user history dictionary is storing words */ /** Locale for which this user history dictionary is storing words */
private final String mLocale; private final String mLocale;
@ -114,6 +116,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
return false; return false;
} }
@Override
protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
return formatVersion >= REQUIRED_BINARY_DICTIONARY_VERSION;
}
@Override
protected String getFileNameExtentionToOpenDict() {
return "/" + FormatSpec.TRIE_FILE_EXTENSION;
}
public void addMultipleDictionaryEntriesToDictionary( public void addMultipleDictionaryEntriesToDictionary(
final ArrayList<LanguageModelParam> languageModelParams, final ArrayList<LanguageModelParam> languageModelParams,
final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {

View File

@ -135,6 +135,12 @@ static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dic
delete dictionary; delete dictionary;
} }
static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
return dictionary->getFormatVersionNumber();
}
static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict, static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict,
jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray, jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray,
jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray, jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray,
@ -428,6 +434,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("(J)V"), const_cast<char *>("(J)V"),
reinterpret_cast<void *>(latinime_BinaryDictionary_close) reinterpret_cast<void *>(latinime_BinaryDictionary_close)
}, },
{
const_cast<char *>("getFormatVersionNative"),
const_cast<char *>("(J)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getFormatVersion)
},
{ {
const_cast<char *>("flushNative"), const_cast<char *>("flushNative"),
const_cast<char *>("(JLjava/lang/String;)V"), const_cast<char *>("(JLjava/lang/String;)V"),

View File

@ -22,6 +22,7 @@
#include "defines.h" #include "defines.h"
#include "jni.h" #include "jni.h"
#include "suggest/core/dictionary/bigram_dictionary.h" #include "suggest/core/dictionary/bigram_dictionary.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/core/suggest_interface.h" #include "suggest/core/suggest_interface.h"
#include "utils/exclusive_ownership_pointer.h" #include "utils/exclusive_ownership_pointer.h"
@ -92,6 +93,11 @@ class Dictionary {
return mDictionaryStructureWithBufferPolicy.get(); return mDictionaryStructureWithBufferPolicy.get();
} }
int getFormatVersionNumber() const {
return mDictionaryStructureWithBufferPolicy.get()->getHeaderStructurePolicy()
->getFormatVersionNumber();
}
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);

View File

@ -29,6 +29,8 @@ class DictionaryHeaderStructurePolicy {
public: public:
virtual ~DictionaryHeaderStructurePolicy() {} virtual ~DictionaryHeaderStructurePolicy() {}
virtual int getFormatVersionNumber() const = 0;
virtual bool supportsDynamicUpdate() const = 0; virtual bool supportsDynamicUpdate() const = 0;
virtual bool requiresGermanUmlautProcessing() const = 0; virtual bool requiresGermanUmlautProcessing() const = 0;

View File

@ -66,6 +66,19 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
~HeaderPolicy() {} ~HeaderPolicy() {}
virtual int getFormatVersionNumber() const {
switch (mDictFormatVersion) {
case FormatUtils::VERSION_2:
return 2;
case FormatUtils::VERSION_3:
return 3;
case FormatUtils::VERSION_4:
return 4;
default:
return 0;
}
}
AK_FORCE_INLINE int getSize() const { AK_FORCE_INLINE int getSize() const {
return mSize; return mSize;
} }

View File

@ -34,6 +34,13 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
FileUtils::getFilePathWithSuffix(dictDirPath, FileUtils::getFilePathWithSuffix(dictDirPath,
DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize, DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize,
tmpDirPath); tmpDirPath);
if (FileUtils::existsDir(tmpDirPath)) {
if (!FileUtils::removeDirAndFiles(tmpDirPath)) {
AKLOGE("Existing directory %s cannot be removed.", tmpDirPath);
ASSERT(false);
return false;
}
}
if (mkdir(tmpDirPath, S_IRWXU) == -1) { if (mkdir(tmpDirPath, S_IRWXU) == -1) {
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false; return false;

View File

@ -41,6 +41,15 @@ namespace latinime {
return static_cast<int>(statBuf.st_size); return static_cast<int>(statBuf.st_size);
} }
/* static */ bool FileUtils::existsDir(const char *const dirPath) {
DIR *const dir = opendir(dirPath);
if (dir == NULL) {
return false;
}
closedir(dir);
return true;
}
// Remove a directory and all files in the directory. // Remove a directory and all files in the directory.
/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) { /* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
DIR *const dir = opendir(dirPath); DIR *const dir = opendir(dirPath);
@ -58,9 +67,11 @@ namespace latinime {
getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath); getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath);
if (remove(filePath) != 0) { if (remove(filePath) != 0) {
AKLOGE("Cannot remove file %s.", filePath); AKLOGE("Cannot remove file %s.", filePath);
closedir(dir);
return false; return false;
} }
} }
closedir(dir);
if (remove(dirPath) != 0) { if (remove(dirPath) != 0) {
AKLOGE("Cannot remove directory %s.", dirPath); AKLOGE("Cannot remove directory %s.", dirPath);
return false; return false;

View File

@ -26,6 +26,8 @@ class FileUtils {
// Returns -1 on error. // Returns -1 on error.
static int getFileSize(const char *const filePath); static int getFileSize(const char *const filePath);
static bool existsDir(const char *const dirPath);
// Remove a directory and all files in the directory. // Remove a directory and all files in the directory.
static bool removeDirAndFiles(const char *const dirPath); static bool removeDirAndFiles(const char *const dirPath);