nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Add version check to open binary dictionary."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2013-11-28 11:52:59 +00:00 committed by Android (Google) Code Review
commit 22e28d5a2d
10 changed files with 97 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -125,6 +125,7 @@ public final class BinaryDictionary extends Dictionary {
private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
private static native void flushWithGCNative(long dict, String filePath);
private static native void closeNative(long dict);
private static native int getFormatVersionNative(long dict);
private static native int getProbabilityNative(long dict, int[] word);
private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1);
private static native int getSuggestionsNative(long dict, long proximityInfo,
@ -241,6 +242,10 @@ public final class BinaryDictionary extends Dictionary {
return mNativeDict != 0;
}
public int getFormatVersion() {
return getFormatVersionNative(mNativeDict);
}
public static float calcNormalizedScore(final String before, final String after,
final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),

36
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View file

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import com.android.inputmethod.latin.utils.StringUtils;
import java.io.File;
import java.util.ArrayList;
@ -128,6 +129,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
protected abstract boolean hasContentChanged();
protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
return true;
}
protected String getFileNameExtentionToOpenDict() {
return "";
}
/**
* Gets the dictionary update controller for the given filename.
*/
@ -238,12 +247,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public void run() {
if (mDictionaryWriter == null) {
mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename + "/"
+ FormatSpec.TRIE_FILE_EXTENSION);
final File file = new File(mContext.getFilesDir(), mFilename);
file.delete();
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
// We have 'fileToOpen' in addition to 'file' for the v4 dictionary format
// where 'file' is a directory, and 'fileToOpen' is a normal file.
final File fileToOpen = new File(mContext.getFilesDir(), mFilename
+ getFileNameExtentionToOpenDict());
// TODO: Make BinaryDictionary's constructor be able to accept filename
// without extension.
mBinaryDictionary = new BinaryDictionary(
file.getAbsolutePath(), 0 /* offset */, file.length(),
fileToOpen.getAbsolutePath(), 0 /* offset */, fileToOpen.length(),
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
} else {
mDictionaryWriter.clear();
@ -482,8 +497,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
+ mFilenameDictionaryUpdateController.mLastUpdateTime);
}
final File file = new File(mContext.getFilesDir(), mFilename + "/"
+ FormatSpec.TRIE_FILE_EXTENSION);
final File file = new File(mContext.getFilesDir(), mFilename
+ getFileNameExtentionToOpenDict());
final String filename = file.getAbsolutePath();
final long length = file.length();
@ -526,8 +541,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
loadDictionaryAsync();
mDictionaryWriter.write(mFilename, getHeaderAttributeMap());
} else {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()
|| !isValidBinaryDictFormatVersion(mBinaryDictionary.getFormatVersion())) {
final File file = new File(mContext.getFilesDir(), mFilename);
file.delete();
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
} else {
@ -623,8 +640,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
// load the shared dictionary.
loadBinaryDictionary();
}
if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
// Binary dictionary is not valid. Regenerate the dictionary file.
if (mBinaryDictionary != null && !(mBinaryDictionary.isValidDictionary()
&& isValidBinaryDictFormatVersion(
mBinaryDictionary.getFormatVersion()))) {
// Binary dictionary or its format version is not valid. Regenerate the
// dictionary file.
mFilenameDictionaryUpdateController.mLastUpdateTime = time;
writeBinaryDictionary();
loadBinaryDictionary();

12
java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
View file

@ -56,6 +56,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
/** Locale for which this user history dictionary is storing words */
private final String mLocale;
@ -114,6 +116,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
return false;
}
@Override
protected boolean isValidBinaryDictFormatVersion(final int formatVersion) {
return formatVersion >= REQUIRED_BINARY_DICTIONARY_VERSION;
}
@Override
protected String getFileNameExtentionToOpenDict() {
return "/" + FormatSpec.TRIE_FILE_EXTENSION;
}
public void addMultipleDictionaryEntriesToDictionary(
final ArrayList<LanguageModelParam> languageModelParams,
final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {

11
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -135,6 +135,12 @@ static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dic
delete dictionary;
}
static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
return dictionary->getFormatVersionNumber();
}
static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict,
jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray,
jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray,
@ -428,6 +434,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("(J)V"),
reinterpret_cast<void *>(latinime_BinaryDictionary_close)
},
{
const_cast<char *>("getFormatVersionNative"),
const_cast<char *>("(J)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getFormatVersion)
},
{
const_cast<char *>("flushNative"),
const_cast<char *>("(JLjava/lang/String;)V"),

6
native/jni/src/suggest/core/dictionary/dictionary.h
View file

@ -22,6 +22,7 @@
#include "defines.h"
#include "jni.h"
#include "suggest/core/dictionary/bigram_dictionary.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/core/suggest_interface.h"
#include "utils/exclusive_ownership_pointer.h"
@ -92,6 +93,11 @@ class Dictionary {
return mDictionaryStructureWithBufferPolicy.get();
}
int getFormatVersionNumber() const {
return mDictionaryStructureWithBufferPolicy.get()->getHeaderStructurePolicy()
->getFormatVersionNumber();
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);

2
native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
View file

@ -29,6 +29,8 @@ class DictionaryHeaderStructurePolicy {
public:
virtual ~DictionaryHeaderStructurePolicy() {}
virtual int getFormatVersionNumber() const = 0;
virtual bool supportsDynamicUpdate() const = 0;
virtual bool requiresGermanUmlautProcessing() const = 0;

13
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View file

@ -66,6 +66,19 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
~HeaderPolicy() {}
virtual int getFormatVersionNumber() const {
switch (mDictFormatVersion) {
case FormatUtils::VERSION_2:
return 2;
case FormatUtils::VERSION_3:
return 3;
case FormatUtils::VERSION_4:
return 4;
default:
return 0;
}
}
AK_FORCE_INLINE int getSize() const {
return mSize;
}

7
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View file

@ -34,6 +34,13 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
FileUtils::getFilePathWithSuffix(dictDirPath,
DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpDirPathBufSize,
tmpDirPath);
if (FileUtils::existsDir(tmpDirPath)) {
if (!FileUtils::removeDirAndFiles(tmpDirPath)) {
AKLOGE("Existing directory %s cannot be removed.", tmpDirPath);
ASSERT(false);
return false;
}
}
if (mkdir(tmpDirPath, S_IRWXU) == -1) {
AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno);
return false;

11
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp
View file

@ -41,6 +41,15 @@ namespace latinime {
return static_cast<int>(statBuf.st_size);
}
/* static */ bool FileUtils::existsDir(const char *const dirPath) {
DIR *const dir = opendir(dirPath);
if (dir == NULL) {
return false;
}
closedir(dir);
return true;
}
// Remove a directory and all files in the directory.
/* static */ bool FileUtils::removeDirAndFiles(const char *const dirPath) {
DIR *const dir = opendir(dirPath);
@ -58,9 +67,11 @@ namespace latinime {
getFilePath(dirPath, dirent->d_name, filePathBufSize, filePath);
if (remove(filePath) != 0) {
AKLOGE("Cannot remove file %s.", filePath);
closedir(dir);
return false;
}
}
closedir(dir);
if (remove(dirPath) != 0) {
AKLOGE("Cannot remove directory %s.", dirPath);
return false;

2
native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.h
View file

@ -26,6 +26,8 @@ class FileUtils {
// Returns -1 on error.
static int getFileSize(const char *const filePath);
static bool existsDir(const char *const dirPath);
// Remove a directory and all files in the directory.
static bool removeDirAndFiles(const char *const dirPath);