Merge "Create empty dictionary file in native code."
commit
e3e10317b0
|
@ -31,6 +31,7 @@ import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implements a static, compacted, binary dictionary of standard words.
|
* Implements a static, compacted, binary dictionary of standard words.
|
||||||
|
@ -104,6 +105,8 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
JniUtils.loadNativeLibrary();
|
JniUtils.loadNativeLibrary();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
|
||||||
|
String[] attributeKeyStringArray, String[] attributeValueStringArray);
|
||||||
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
|
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
|
||||||
boolean isUpdatable);
|
boolean isUpdatable);
|
||||||
private static native void flushNative(long dict, String filePath);
|
private static native void flushNative(long dict, String filePath);
|
||||||
|
@ -127,6 +130,20 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
private static native int calculateProbabilityNative(long dict, int unigramProbability,
|
private static native int calculateProbabilityNative(long dict, int unigramProbability,
|
||||||
int bigramProbability);
|
int bigramProbability);
|
||||||
|
|
||||||
|
@UsedForTesting
|
||||||
|
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
|
||||||
|
final Map<String, String> attributeMap) {
|
||||||
|
final String[] keyArray = new String[attributeMap.size()];
|
||||||
|
final String[] valueArray = new String[attributeMap.size()];
|
||||||
|
int index = 0;
|
||||||
|
for (final String key : attributeMap.keySet()) {
|
||||||
|
keyArray[index] = key;
|
||||||
|
valueArray[index] = attributeMap.get(key);
|
||||||
|
index++;
|
||||||
|
}
|
||||||
|
return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: Move native dict into session
|
// TODO: Move native dict into session
|
||||||
private final void loadDictionary(final String path, final long startOffset,
|
private final void loadDictionary(final String path, final long startOffset,
|
||||||
final long length, final boolean isUpdatable) {
|
final long length, final boolean isUpdatable) {
|
||||||
|
|
|
@ -22,12 +22,7 @@ import android.util.Log;
|
||||||
|
|
||||||
import com.android.inputmethod.annotations.UsedForTesting;
|
import com.android.inputmethod.annotations.UsedForTesting;
|
||||||
import com.android.inputmethod.keyboard.ProximityInfo;
|
import com.android.inputmethod.keyboard.ProximityInfo;
|
||||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
|
||||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
|
||||||
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
|
import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
|
||||||
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
|
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
|
||||||
import com.android.inputmethod.latin.utils.AsyncResultHolder;
|
import com.android.inputmethod.latin.utils.AsyncResultHolder;
|
||||||
|
@ -35,9 +30,9 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||||
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
|
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
@ -68,8 +63,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
*/
|
*/
|
||||||
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
|
protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
|
||||||
|
|
||||||
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
|
private static final int DICTIONARY_FORMAT_VERSION = 3;
|
||||||
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
|
|
||||||
|
private static final String SUPPORTS_DYNAMIC_UPDATE =
|
||||||
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A static map of time recorders, each of which records the time of accesses to a single binary
|
* A static map of time recorders, each of which records the time of accesses to a single binary
|
||||||
|
@ -233,6 +230,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected Map<String, String> getHeaderAttributeMap() {
|
||||||
|
HashMap<String, String> attributeMap = new HashMap<String, String>();
|
||||||
|
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
||||||
|
SUPPORTS_DYNAMIC_UPDATE);
|
||||||
|
return attributeMap;
|
||||||
|
}
|
||||||
|
|
||||||
protected void clear() {
|
protected void clear() {
|
||||||
getExecutor(mFilename).execute(new Runnable() {
|
getExecutor(mFilename).execute(new Runnable() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -240,17 +244,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
|
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
|
||||||
mBinaryDictionary.close();
|
mBinaryDictionary.close();
|
||||||
final File file = new File(mContext.getFilesDir(), mFilename);
|
final File file = new File(mContext.getFilesDir(), mFilename);
|
||||||
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
|
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
|
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
|
||||||
false, false));
|
|
||||||
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
|
|
||||||
try {
|
|
||||||
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
|
|
||||||
} catch (IOException e) {
|
|
||||||
Log.e(TAG, "Exception in creating new dictionary file.", e);
|
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
Log.e(TAG, "Exception in creating new dictionary file.", e);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
mDictionaryWriter.clear();
|
mDictionaryWriter.clear();
|
||||||
}
|
}
|
||||||
|
@ -498,17 +493,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
|
if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
|
||||||
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
|
if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
|
||||||
final File file = new File(mContext.getFilesDir(), mFilename);
|
final File file = new File(mContext.getFilesDir(), mFilename);
|
||||||
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
|
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
|
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
|
||||||
false, false));
|
|
||||||
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
|
|
||||||
try {
|
|
||||||
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
|
|
||||||
} catch (IOException e) {
|
|
||||||
Log.e(TAG, "Exception in creating new dictionary file.", e);
|
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
Log.e(TAG, "Exception in creating new dictionary file.", e);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if (mBinaryDictionary.needsToRunGC()) {
|
if (mBinaryDictionary.needsToRunGC()) {
|
||||||
mBinaryDictionary.flushWithGC();
|
mBinaryDictionary.flushWithGC();
|
||||||
|
|
|
@ -325,6 +325,12 @@ public final class FormatSpec {
|
||||||
public final int mHeaderSize;
|
public final int mHeaderSize;
|
||||||
public final DictionaryOptions mDictionaryOptions;
|
public final DictionaryOptions mDictionaryOptions;
|
||||||
public final FormatOptions mFormatOptions;
|
public final FormatOptions mFormatOptions;
|
||||||
|
// Note that these are corresponding definitions in native code in latinime::HeaderPolicy
|
||||||
|
// and latinime::HeaderReadWriteUtils.
|
||||||
|
public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
|
||||||
|
public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
|
||||||
|
public static final String ATTRIBUTE_VALUE_TRUE = "1";
|
||||||
|
|
||||||
private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
|
private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
|
||||||
private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
|
private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
|
||||||
private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
|
private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
|
||||||
|
|
|
@ -34,12 +34,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class is a base class of a dictionary for the personalized prediction language model.
|
* This class is a base class of a dictionary that supports decaying for the personalized language
|
||||||
|
* model.
|
||||||
*/
|
*/
|
||||||
public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {
|
public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
|
||||||
private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();
|
private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
|
||||||
public static final boolean DBG_SAVE_RESTORE = false;
|
public static final boolean DBG_SAVE_RESTORE = false;
|
||||||
private static final boolean DBG_STRESS_TEST = false;
|
private static final boolean DBG_STRESS_TEST = false;
|
||||||
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
|
private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
|
||||||
|
@ -60,8 +63,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
|
||||||
// Should always be false except when we use this class for test
|
// Should always be false except when we use this class for test
|
||||||
@UsedForTesting boolean mIsTest = false;
|
@UsedForTesting boolean mIsTest = false;
|
||||||
|
|
||||||
/* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,
|
/* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
|
||||||
final SharedPreferences sp, final String dictionaryType, final String fileName) {
|
final String locale, final SharedPreferences sp, final String dictionaryType,
|
||||||
|
final String fileName) {
|
||||||
super(context, fileName, dictionaryType, true);
|
super(context, fileName, dictionaryType, true);
|
||||||
mLocale = locale;
|
mLocale = locale;
|
||||||
mFileName = fileName;
|
mFileName = fileName;
|
||||||
|
@ -83,6 +87,16 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
|
||||||
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
|
Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Map<String, String> getHeaderAttributeMap() {
|
||||||
|
HashMap<String, String> attributeMap = new HashMap<String, String>();
|
||||||
|
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
||||||
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
|
attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
|
||||||
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
|
return attributeMap;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean hasContentChanged() {
|
protected boolean hasContentChanged() {
|
||||||
return false;
|
return false;
|
|
@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
|
|
||||||
// TODO: Use a dynamic binary dictionary instead
|
// TODO: Use a dynamic binary dictionary instead
|
||||||
public WeakReference<PersonalizationDictionary> mDictionary;
|
public WeakReference<PersonalizationDictionary> mDictionary;
|
||||||
public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary;
|
public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
|
||||||
public final String mSystemLocale;
|
public final String mSystemLocale;
|
||||||
public PersonalizationDictionaryUpdateSession(String locale) {
|
public PersonalizationDictionaryUpdateSession(String locale) {
|
||||||
mSystemLocale = locale;
|
mSystemLocale = locale;
|
||||||
|
@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
|
mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) {
|
public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
|
||||||
mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);
|
mPredictionDictionary =
|
||||||
|
new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected PersonalizationDictionary getDictionary() {
|
protected PersonalizationDictionary getDictionary() {
|
||||||
return mDictionary == null ? null : mDictionary.get();
|
return mDictionary == null ? null : mDictionary.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected DynamicPredictionDictionaryBase getPredictionDictionary() {
|
protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
|
||||||
return mPredictionDictionary == null ? null : mPredictionDictionary.get();
|
return mPredictionDictionary == null ? null : mPredictionDictionary.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void unsetPredictionDictionary() {
|
private void unsetPredictionDictionary() {
|
||||||
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
|
final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
|
||||||
if (dictionary == null) {
|
if (dictionary == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void clearAndFlushPredictionDictionary(Context context) {
|
public void clearAndFlushPredictionDictionary(Context context) {
|
||||||
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
|
final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
|
||||||
if (dictionary == null) {
|
if (dictionary == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -105,7 +106,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
// TODO: Support multi locale to add bigram
|
// TODO: Support multi locale to add bigram
|
||||||
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
|
public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
|
||||||
int frequency) {
|
int frequency) {
|
||||||
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
|
final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
|
||||||
if (dictionary == null) {
|
if (dictionary == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -116,7 +117,7 @@ public abstract class PersonalizationDictionaryUpdateSession {
|
||||||
// TODO: Support multi locale to add bigram
|
// TODO: Support multi locale to add bigram
|
||||||
public void addBigramsToPersonalizationDictionary(
|
public void addBigramsToPersonalizationDictionary(
|
||||||
final ArrayList<PersonalizationLanguageModelParam> lmParams) {
|
final ArrayList<PersonalizationLanguageModelParam> lmParams) {
|
||||||
final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();
|
final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
|
||||||
if (dictionary == null) {
|
if (dictionary == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary;
|
||||||
import android.content.Context;
|
import android.content.Context;
|
||||||
import android.content.SharedPreferences;
|
import android.content.SharedPreferences;
|
||||||
|
|
||||||
public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase {
|
public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
|
||||||
private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
|
private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
|
||||||
|
|
||||||
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
|
/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
|
||||||
|
|
|
@ -54,7 +54,7 @@ public final class UserHistoryDictionaryBigramList {
|
||||||
* Called when loaded from the SQL DB.
|
* Called when loaded from the SQL DB.
|
||||||
*/
|
*/
|
||||||
public void addBigram(String word1, String word2, byte fcValue) {
|
public void addBigram(String word1, String word2, byte fcValue) {
|
||||||
if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
|
if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
|
||||||
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
|
Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue);
|
||||||
}
|
}
|
||||||
final HashMap<String, Byte> map;
|
final HashMap<String, Byte> map;
|
||||||
|
@ -74,7 +74,7 @@ public final class UserHistoryDictionaryBigramList {
|
||||||
* Called when inserted to the SQL DB.
|
* Called when inserted to the SQL DB.
|
||||||
*/
|
*/
|
||||||
public void updateBigram(String word1, String word2, byte fcValue) {
|
public void updateBigram(String word1, String word2, byte fcValue) {
|
||||||
if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) {
|
if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) {
|
||||||
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
|
Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue);
|
||||||
}
|
}
|
||||||
final HashMap<String, Byte> map;
|
final HashMap<String, Byte> map;
|
||||||
|
|
|
@ -26,7 +26,7 @@ import android.content.SharedPreferences;
|
||||||
* Locally gathers stats about the words user types and various other signals like auto-correction
|
* Locally gathers stats about the words user types and various other signals like auto-correction
|
||||||
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
|
* cancellation or manual picks. This allows the keyboard to adapt to the typist over time.
|
||||||
*/
|
*/
|
||||||
public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase {
|
public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
|
||||||
/* package for tests */ static final String NAME =
|
/* package for tests */ static final String NAME =
|
||||||
UserHistoryPredictionDictionary.class.getSimpleName();
|
UserHistoryPredictionDictionary.class.getSimpleName();
|
||||||
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
|
/* package */ UserHistoryPredictionDictionary(final Context context, final String locale,
|
||||||
|
|
|
@ -26,12 +26,55 @@
|
||||||
#include "suggest/core/dictionary/dictionary.h"
|
#include "suggest/core/dictionary/dictionary.h"
|
||||||
#include "suggest/core/suggest_options.h"
|
#include "suggest/core/suggest_options.h"
|
||||||
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
|
#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
|
||||||
|
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
|
||||||
#include "utils/autocorrection_threshold_utils.h"
|
#include "utils/autocorrection_threshold_utils.h"
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
class ProximityInfo;
|
class ProximityInfo;
|
||||||
|
|
||||||
|
// TODO: Move to makedict.
|
||||||
|
static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
|
||||||
|
jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
|
||||||
|
jobjectArray attributeValueStringArray) {
|
||||||
|
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
|
||||||
|
char filePathChars[filePathUtf8Length + 1];
|
||||||
|
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
|
||||||
|
filePathChars[filePathUtf8Length] = '\0';
|
||||||
|
|
||||||
|
const int keyCount = env->GetArrayLength(attributeKeyStringArray);
|
||||||
|
const int valueCount = env->GetArrayLength(attributeValueStringArray);
|
||||||
|
if (keyCount != valueCount) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
HeaderReadWriteUtils::AttributeMap attributeMap;
|
||||||
|
for (int i = 0; i < keyCount; i++) {
|
||||||
|
jstring keyString = static_cast<jstring>(
|
||||||
|
env->GetObjectArrayElement(attributeKeyStringArray, i));
|
||||||
|
const jsize keyUtf8Length = env->GetStringUTFLength(keyString);
|
||||||
|
char keyChars[keyUtf8Length + 1];
|
||||||
|
env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars);
|
||||||
|
keyChars[keyUtf8Length] = '\0';
|
||||||
|
HeaderReadWriteUtils::AttributeMap::key_type key;
|
||||||
|
HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key);
|
||||||
|
|
||||||
|
jstring valueString = static_cast<jstring>(
|
||||||
|
env->GetObjectArrayElement(attributeValueStringArray, i));
|
||||||
|
const jsize valueUtf8Length = env->GetStringUTFLength(valueString);
|
||||||
|
char valueChars[valueUtf8Length + 1];
|
||||||
|
env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars);
|
||||||
|
valueChars[valueUtf8Length] = '\0';
|
||||||
|
HeaderReadWriteUtils::AttributeMap::mapped_type value;
|
||||||
|
HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value);
|
||||||
|
|
||||||
|
attributeMap[key] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
|
||||||
|
&attributeMap);
|
||||||
|
}
|
||||||
|
|
||||||
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
|
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
|
||||||
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
|
jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
|
||||||
PROF_OPEN;
|
PROF_OPEN;
|
||||||
|
@ -281,6 +324,11 @@ static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jcl
|
||||||
}
|
}
|
||||||
|
|
||||||
static const JNINativeMethod sMethods[] = {
|
static const JNINativeMethod sMethods[] = {
|
||||||
|
{
|
||||||
|
const_cast<char *>("createEmptyDictFileNative"),
|
||||||
|
const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
|
||||||
|
reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
|
||||||
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("openNative"),
|
const_cast<char *>("openNative"),
|
||||||
const_cast<char *>("(Ljava/lang/String;JJZ)J"),
|
const_cast<char *>("(Ljava/lang/String;JJZ)J"),
|
||||||
|
|
|
@ -22,6 +22,8 @@
|
||||||
|
|
||||||
namespace latinime {
|
namespace latinime {
|
||||||
|
|
||||||
|
|
||||||
|
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
|
||||||
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
|
const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
|
||||||
const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
|
const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE";
|
||||||
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
|
const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
|
||||||
|
|
|
@ -45,6 +45,7 @@ const HeaderReadWriteUtils::DictionaryFlags
|
||||||
const HeaderReadWriteUtils::DictionaryFlags
|
const HeaderReadWriteUtils::DictionaryFlags
|
||||||
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
|
HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
|
||||||
|
|
||||||
|
// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
|
||||||
const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
|
const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
|
||||||
const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
|
const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
|
||||||
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
|
"REQUIRES_GERMAN_UMLAUT_PROCESSING";
|
||||||
|
|
|
@ -21,24 +21,18 @@ import android.test.suitebuilder.annotation.LargeTest;
|
||||||
import android.util.Pair;
|
import android.util.Pair;
|
||||||
|
|
||||||
import com.android.inputmethod.latin.makedict.CodePointUtils;
|
import com.android.inputmethod.latin.makedict.CodePointUtils;
|
||||||
import com.android.inputmethod.latin.makedict.DictEncoder;
|
|
||||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
|
||||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
|
||||||
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
|
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
|
||||||
@LargeTest
|
@LargeTest
|
||||||
public class BinaryDictionaryTests extends AndroidTestCase {
|
public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
|
|
||||||
new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
|
|
||||||
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
|
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
|
||||||
private static final String TEST_LOCALE = "test";
|
private static final String TEST_LOCALE = "test";
|
||||||
|
|
||||||
|
@ -52,15 +46,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
private File createEmptyDictionaryAndGetFile(final String filename) throws IOException,
|
private File createEmptyDictionaryAndGetFile(final String filename) throws IOException {
|
||||||
UnsupportedFormatException {
|
|
||||||
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
|
|
||||||
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
|
|
||||||
final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION,
|
final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION,
|
||||||
getContext().getCacheDir());
|
getContext().getCacheDir());
|
||||||
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
|
Map<String, String> attributeMap = new HashMap<String, String>();
|
||||||
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
|
attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
|
||||||
return file;
|
FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
|
||||||
|
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
|
||||||
|
3 /* dictVersion */, attributeMap)) {
|
||||||
|
return file;
|
||||||
|
} else {
|
||||||
|
throw new IOException("Empty dictionary cannot be created.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testIsValidDictionary() {
|
public void testIsValidDictionary() {
|
||||||
|
@ -69,8 +66,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -95,8 +90,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -139,8 +132,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -169,8 +160,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -240,8 +229,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -294,8 +281,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -342,8 +327,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -392,8 +375,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
|
||||||
|
@ -445,8 +426,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
|
@ -516,8 +495,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
|
||||||
|
@ -617,8 +594,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail("IOException while writing an initial dictionary : " + e);
|
fail("IOException while writing an initial dictionary : " + e);
|
||||||
} catch (UnsupportedFormatException e) {
|
|
||||||
fail("UnsupportedFormatException while writing an initial dictionary : " + e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
final ArrayList<String> words = new ArrayList<String>();
|
final ArrayList<String> words = new ArrayList<String>();
|
||||||
|
|
Loading…
Reference in New Issue