am fef9e1cd: am 0ae51a5e: am e3e10317: Merge "Create empty dictionary file in native code."

* commit 'fef9e1cdb374354957e8be774a4e9099a05f2335':
  Create empty dictionary file in native code.
main
Keisuke Kuroyanagi 2013-10-03 13:21:22 -07:00 committed by Android Git Automerger
commit 3e313e3db7
12 changed files with 133 additions and 83 deletions

View File

@ -31,6 +31,7 @@ import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
/** /**
* Implements a static, compacted, binary dictionary of standard words. * Implements a static, compacted, binary dictionary of standard words.
@ -104,6 +105,8 @@ public final class BinaryDictionary extends Dictionary {
JniUtils.loadNativeLibrary(); JniUtils.loadNativeLibrary();
} }
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize, private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable); boolean isUpdatable);
private static native void flushNative(long dict, String filePath); private static native void flushNative(long dict, String filePath);
@ -127,6 +130,20 @@ public final class BinaryDictionary extends Dictionary {
private static native int calculateProbabilityNative(long dict, int unigramProbability, private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability); int bigramProbability);
@UsedForTesting
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
for (final String key : attributeMap.keySet()) {
keyArray[index] = key;
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
}
// TODO: Move native dict into session // TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset, private final void loadDictionary(final String path, final long startOffset,
final long length, final boolean isUpdatable) { final long length, final boolean isUpdatable) {

View File

@ -22,12 +22,7 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.AsyncResultHolder;
@ -35,9 +30,9 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
@ -69,8 +64,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/ */
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS = private static final int DICTIONARY_FORMAT_VERSION = 3;
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
private static final String SUPPORTS_DYNAMIC_UPDATE =
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;
/** /**
* A static map of update controllers, each of which records the time of accesses to a single * A static map of update controllers, each of which records the time of accesses to a single
@ -235,6 +232,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}); });
} }
protected Map<String, String> getHeaderAttributeMap() {
HashMap<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
SUPPORTS_DYNAMIC_UPDATE);
return attributeMap;
}
protected void clear() { protected void clear() {
getExecutor(mFilename).execute(new Runnable() { getExecutor(mFilename).execute(new Runnable() {
@Override @Override
@ -242,17 +246,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
mBinaryDictionary.close(); mBinaryDictionary.close();
final File file = new File(mContext.getFilesDir(), mFilename); final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else { } else {
mDictionaryWriter.clear(); mDictionaryWriter.clear();
} }
@ -506,17 +501,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
final File file = new File(mContext.getFilesDir(), mFilename); final File file = new File(mContext.getFilesDir(), mFilename);
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
false, false));
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
try {
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
} catch (IOException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Exception in creating new dictionary file.", e);
}
} else { } else {
if (mBinaryDictionary.needsToRunGC()) { if (mBinaryDictionary.needsToRunGC()) {
mBinaryDictionary.flushWithGC(); mBinaryDictionary.flushWithGC();

View File

@ -325,6 +325,12 @@ public final class FormatSpec {
public final int mHeaderSize; public final int mHeaderSize;
public final DictionaryOptions mDictionaryOptions; public final DictionaryOptions mDictionaryOptions;
public final FormatOptions mFormatOptions; public final FormatOptions mFormatOptions;
// Note that these are corresponding definitions in native code in latinime::HeaderPolicy
// and latinime::HeaderReadWriteUtils.
public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
public static final String ATTRIBUTE_VALUE_TRUE = "1";
private static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";

View File

@ -34,12 +34,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
/** /**
* This class is a base class of a dictionary for the personalized prediction language model. * This class is a base class of a dictionary that supports decaying for the personalized language
* model.
*/ */
public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary { public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName(); private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
public static final boolean DBG_SAVE_RESTORE = false; public static final boolean DBG_SAVE_RESTORE = false;
private static final boolean DBG_STRESS_TEST = false; private static final boolean DBG_STRESS_TEST = false;
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
@ -60,8 +63,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Should always be false except when we use this class for test // Should always be false except when we use this class for test
@UsedForTesting boolean mIsTest = false; @UsedForTesting boolean mIsTest = false;
/* package */ DynamicPredictionDictionaryBase(final Context context, final String locale, /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
final SharedPreferences sp, final String dictionaryType, final String fileName) { final String locale, final SharedPreferences sp, final String dictionaryType,
final String fileName) {
super(context, fileName, dictionaryType, true); super(context, fileName, dictionaryType, true);
mLocale = locale; mLocale = locale;
mFileName = fileName; mFileName = fileName;
@ -83,6 +87,16 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
} }
@Override
protected Map<String, String> getHeaderAttributeMap() {
HashMap<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
return attributeMap;
}
@Override @Override
protected boolean hasContentChanged() { protected boolean hasContentChanged() {
return false; return false;

View File

@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Use a dynamic binary dictionary instead // TODO: Use a dynamic binary dictionary instead
public WeakReference<PersonalizationDictionary> mDictionary; public WeakReference<PersonalizationDictionary> mDictionary;
public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary; public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
public final String mSystemLocale; public final String mSystemLocale;
public PersonalizationDictionaryUpdateSession(String locale) { public PersonalizationDictionaryUpdateSession(String locale) {
mSystemLocale = locale; mSystemLocale = locale;
@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession {
mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
} }
public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) { public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); mPredictionDictionary =
new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
} }
protected PersonalizationDictionary getDictionary() { protected PersonalizationDictionary getDictionary() {
return mDictionary == null ? null : mDictionary.get(); return mDictionary == null ? null : mDictionary.get();
} }
protected DynamicPredictionDictionaryBase getPredictionDictionary() { protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
return mPredictionDictionary == null ? null : mPredictionDictionary.get(); return mPredictionDictionary == null ? null : mPredictionDictionary.get();
} }
@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
} }
private void unsetPredictionDictionary() { private void unsetPredictionDictionary() {
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) { if (dictionary == null) {
return; return;
} }
@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
} }
public void clearAndFlushPredictionDictionary(Context context) { public void clearAndFlushPredictionDictionary(Context context) {
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) { if (dictionary == null) {
return; return;
} }
@ -105,7 +106,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Support multi locale to add bigram // TODO: Support multi locale to add bigram
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
int frequency) { int frequency) {
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) { if (dictionary == null) {
return; return;
} }
@ -116,7 +117,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
// TODO: Support multi locale to add bigram // TODO: Support multi locale to add bigram
public void addBigramsToPersonalizationDictionary( public void addBigramsToPersonalizationDictionary(
final ArrayList<PersonalizationLanguageModelParam> lmParams) { final ArrayList<PersonalizationLanguageModelParam> lmParams) {
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
if (dictionary == null) { if (dictionary == null) {
return; return;
} }

View File

@ -22,7 +22,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import android.content.Context; import android.content.Context;
import android.content.SharedPreferences; import android.content.SharedPreferences;
public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase { public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale, /* package */ PersonalizationPredictionDictionary(final Context context, final String locale,

View File

@ -54,7 +54,7 @@ public final class UserHistoryDictionaryBigramList {
* Called when loaded from the SQL DB. * Called when loaded from the SQL DB.
*/ */
public void addBigram(String word1, String word2, byte fcValue) { public void addBigram(String word1, String word2, byte fcValue) {
if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
} }
final HashMap<String, Byte> map; final HashMap<String, Byte> map;
@ -74,7 +74,7 @@ public final class UserHistoryDictionaryBigramList {
* Called when inserted to the SQL DB. * Called when inserted to the SQL DB.
*/ */
public void updateBigram(String word1, String word2, byte fcValue) { public void updateBigram(String word1, String word2, byte fcValue) {
if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
} }
final HashMap<String, Byte> map; final HashMap<String, Byte> map;

View File

@ -26,7 +26,7 @@ import android.content.SharedPreferences;
* Locally gathers stats about the words user types and various other signals like auto-correction * Locally gathers stats about the words user types and various other signals like auto-correction
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time. * cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
*/ */
public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase { public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
/* package for tests */ static final String NAME = /* package for tests */ static final String NAME =
UserHistoryPredictionDictionary.class.getSimpleName(); UserHistoryPredictionDictionary.class.getSimpleName();
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale, /* package */ UserHistoryPredictionDictionary(final Context context, final String locale,

View File

@ -26,12 +26,55 @@
#include "suggest/core/dictionary/dictionary.h" #include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/suggest_options.h" #include "suggest/core/suggest_options.h"
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h" #include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/autocorrection_threshold_utils.h" #include "utils/autocorrection_threshold_utils.h"
namespace latinime { namespace latinime {
class ProximityInfo; class ProximityInfo;
// TODO: Move to makedict.
static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
jobjectArray attributeValueStringArray) {
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
char filePathChars[filePathUtf8Length + 1];
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
filePathChars[filePathUtf8Length] = '\0';
const int keyCount = env->GetArrayLength(attributeKeyStringArray);
const int valueCount = env->GetArrayLength(attributeValueStringArray);
if (keyCount != valueCount) {
return false;
}
HeaderReadWriteUtils::AttributeMap attributeMap;
for (int i = 0; i < keyCount; i++) {
jstring keyString = static_cast<jstring>(
env->GetObjectArrayElement(attributeKeyStringArray, i));
const jsize keyUtf8Length = env->GetStringUTFLength(keyString);
char keyChars[keyUtf8Length + 1];
env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars);
keyChars[keyUtf8Length] = '\0';
HeaderReadWriteUtils::AttributeMap::key_type key;
HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key);
jstring valueString = static_cast<jstring>(
env->GetObjectArrayElement(attributeValueStringArray, i));
const jsize valueUtf8Length = env->GetStringUTFLength(valueString);
char valueChars[valueUtf8Length + 1];
env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars);
valueChars[valueUtf8Length] = '\0';
HeaderReadWriteUtils::AttributeMap::mapped_type value;
HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value);
attributeMap[key] = value;
}
return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
&attributeMap);
}
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir, static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
jlong dictOffset, jlong dictSize, jboolean isUpdatable) { jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
PROF_OPEN; PROF_OPEN;
@ -281,6 +324,11 @@ static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jcl
} }
static const JNINativeMethod sMethods[] = { static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
},
{ {
const_cast<char *>("openNative"), const_cast<char *>("openNative"),
const_cast<char *>("(Ljava/lang/String;JJZ)J"), const_cast<char *>("(Ljava/lang/String;JJZ)J"),

View File

@ -22,6 +22,8 @@
namespace latinime { namespace latinime {
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";

View File

@ -45,6 +45,7 @@ const HeaderReadWriteUtils::DictionaryFlags
const HeaderReadWriteUtils::DictionaryFlags const HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE"; const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
"REQUIRES_GERMAN_UMLAUT_PROCESSING"; "REQUIRES_GERMAN_UMLAUT_PROCESSING";

View File

@ -21,24 +21,18 @@ import android.test.suitebuilder.annotation.LargeTest;
import android.util.Pair; import android.util.Pair;
import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.Random; import java.util.Random;
@LargeTest @LargeTest
public class BinaryDictionaryTests extends AndroidTestCase { public class BinaryDictionaryTests extends AndroidTestCase {
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final String TEST_LOCALE = "test"; private static final String TEST_LOCALE = "test";
@ -52,15 +46,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
super.tearDown(); super.tearDown();
} }
private File createEmptyDictionaryAndGetFile(final String filename) throws IOException, private File createEmptyDictionaryAndGetFile(final String filename) throws IOException {
UnsupportedFormatException {
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir()); getContext().getCacheDir());
final DictEncoder dictEncoder = new Ver3DictEncoder(file); Map<String, String> attributeMap = new HashMap<String, String>();
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
return file; FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
3 /* dictVersion */, attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary cannot be created.");
}
} }
public void testIsValidDictionary() { public void testIsValidDictionary() {
@ -69,8 +66,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -95,8 +90,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -139,8 +132,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -169,8 +160,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -240,8 +229,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -294,8 +281,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -342,8 +327,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -392,8 +375,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */, 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
@ -445,8 +426,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
@ -516,8 +495,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
@ -617,8 +594,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
} }
final ArrayList<String> words = new ArrayList<String>(); final ArrayList<String> words = new ArrayList<String>();