Merge "Remove corrupted ver4 dictionaries."

main
Keisuke Kuroyanagi 2014-02-21 09:03:19 +00:00 committed by Android (Google) Code Review
commit ad508d029f
3 changed files with 51 additions and 10 deletions

View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin; package com.android.inputmethod.latin;
import android.text.TextUtils; import android.text.TextUtils;
import android.util.Log;
import android.util.SparseArray; import android.util.SparseArray;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
@ -29,6 +30,7 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.settings.NativeSuggestOptions;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.StringUtils; import com.android.inputmethod.latin.utils.StringUtils;
@ -84,6 +86,7 @@ public final class BinaryDictionary extends Dictionary {
private final Locale mLocale; private final Locale mLocale;
private final long mDictSize; private final long mDictSize;
private final String mDictFilePath; private final String mDictFilePath;
private final boolean mIsUpdatable;
private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH]; private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH];
private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS]; private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS];
private final int[] mSpaceIndices = new int[MAX_RESULTS]; private final int[] mSpaceIndices = new int[MAX_RESULTS];
@ -130,6 +133,7 @@ public final class BinaryDictionary extends Dictionary {
mLocale = locale; mLocale = locale;
mDictSize = length; mDictSize = length;
mDictFilePath = filename; mDictFilePath = filename;
mIsUpdatable = isUpdatable;
mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance); mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance);
loadDictionary(filename, offset, length, isUpdatable); loadDictionary(filename, offset, length, isUpdatable);
} }
@ -177,6 +181,7 @@ public final class BinaryDictionary extends Dictionary {
int bigramProbability); int bigramProbability);
private static native int setCurrentTimeForTestNative(int currentTime); private static native int setCurrentTimeForTestNative(int currentTime);
private static native String getPropertyNative(long dict, String query); private static native String getPropertyNative(long dict, String query);
private static native boolean isCorruptedNative(long dict);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion, public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) { final Locale locale, final Map<String, String> attributeMap) {
@ -198,6 +203,22 @@ public final class BinaryDictionary extends Dictionary {
mNativeDict = openNative(path, startOffset, length, isUpdatable); mNativeDict = openNative(path, startOffset, length, isUpdatable);
} }
// TODO: Check isCorrupted() for main dictionaries.
public boolean isCorrupted() {
if (!isValidDictionary()) {
return false;
}
if (!isCorruptedNative(mNativeDict)) {
return false;
}
// TODO: Record the corruption.
Log.e(TAG, "BinaryDictionary (" + mDictFilePath + ") is corrupted.");
Log.e(TAG, "locale: " + mLocale);
Log.e(TAG, "dict size: " + mDictSize);
Log.e(TAG, "updatable: " + mIsUpdatable);
return true;
}
@UsedForTesting @UsedForTesting
public DictionaryHeader getHeader() throws UnsupportedFormatException { public DictionaryHeader getHeader() throws UnsupportedFormatException {
if (mNativeDict == 0) { if (mNativeDict == 0) {
@ -444,7 +465,7 @@ public final class BinaryDictionary extends Dictionary {
// only be called for actual files. Right now it's only called by the flush() family of // only be called for actual files. Right now it's only called by the flush() family of
// functions, which require an updatable dictionary, so it's okay. But beware. // functions, which require an updatable dictionary, so it's okay. But beware.
loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */,
dictFile.length(), true /* isUpdatable */); dictFile.length(), mIsUpdatable);
} }
public void flush() { public void flush() {

View File

@ -276,22 +276,26 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return attributeMap; return attributeMap;
} }
private void removeBinaryDictionaryLocked() {
if (mBinaryDictionary != null) {
mBinaryDictionary.close();
}
if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) {
Log.e(TAG, "Can't remove a file: " + mDictFile.getName());
}
mBinaryDictionary = null;
}
protected void clear() { protected void clear() {
final File dictFile = mDictFile;
getExecutor(mDictName).execute(new Runnable() { getExecutor(mDictName).execute(new Runnable() {
@Override @Override
public void run() { public void run() {
if (mDictionaryWriter == null) { if (mDictionaryWriter == null) {
if (mBinaryDictionary != null) { removeBinaryDictionaryLocked();
mBinaryDictionary.close(); BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
}
if (dictFile.exists() && !FileUtils.deleteRecursively(dictFile)) {
Log.e(TAG, "Can't remove a file: " + dictFile.getName());
}
BinaryDictionary.createEmptyDictFile(dictFile.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
mBinaryDictionary = new BinaryDictionary( mBinaryDictionary = new BinaryDictionary(
dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(),
true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable); true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
} else { } else {
mDictionaryWriter.clear(); mDictionaryWriter.clear();
@ -469,6 +473,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
proximityInfo, blockOffensiveWords, additionalFeaturesOptions, proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
sessionId); sessionId);
holder.set(binarySuggestion); holder.set(binarySuggestion);
if (mBinaryDictionary.isCorrupted()) {
removeBinaryDictionaryLocked();
}
} }
}); });
return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);

View File

@ -529,6 +529,14 @@ static int latinime_BinaryDictionary_setCurrentTimeForTest(JNIEnv *env, jclass c
return TimeKeeper::peekCurrentTime(); return TimeKeeper::peekCurrentTime();
} }
static bool latinime_BinaryDictionary_isCorruptedNative(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) {
return false;
}
return dictionary->getDictionaryStructurePolicy()->isCorrupted();
}
static const JNINativeMethod sMethods[] = { static const JNINativeMethod sMethods[] = {
{ {
const_cast<char *>("createEmptyDictFileNative"), const_cast<char *>("createEmptyDictFileNative"),
@ -642,6 +650,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("getPropertyNative"), const_cast<char *>("getPropertyNative"),
const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"), const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty) reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty)
},
{
const_cast<char *>("isCorruptedNative"),
const_cast<char *>("(J)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_isCorruptedNative)
} }
}; };