nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Separate utility methods from BinaryDictionary. 

Bug: 8187060
Change-Id: Ice2984e332b7bd3bb17174aefc80b5635b72fc50
main
Keisuke Kuroyanagi 2014-03-05 18:19:34 +09:00
parent 9e51c6c0c6
commit e784148ae6
14 changed files with 122 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -40,7 +40,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
/**
* Implements a static, compacted, binary dictionary of standard words.
@ -142,8 +141,6 @@ public final class BinaryDictionary extends Dictionary {
JniUtils.loadNativeLibrary();
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
@ -167,8 +164,6 @@ public final class BinaryDictionary extends Dictionary {
int[] suggestOptions, int[] prevWordCodePointArray,
int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes,
int[] outputAutoCommitFirstWordConfidence);
private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
private static native int editDistanceNative(int[] before, int[] after);
private static native void addUnigramWordNative(long dict, int[] word, int probability,
int[] shortcutTarget, int shortcutProbability, boolean isNotAWord,
boolean isBlacklisted, int timestamp);
@ -179,24 +174,9 @@ public final class BinaryDictionary extends Dictionary {
LanguageModelParam[] languageModelParams, int startIndex);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
private static native int setCurrentTimeForTestNative(int currentTime);
private static native String getPropertyNative(long dict, String query);
private static native boolean isCorruptedNative(long dict);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
for (final String key : attributeMap.keySet()) {
keyArray[index] = key;
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
final long length, final boolean isUpdatable) {
@ -323,20 +303,6 @@ public final class BinaryDictionary extends Dictionary {
return getFormatVersionNative(mNativeDict);
}
public static float calcNormalizedScore(final String before, final String after,
final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after), score);
}
public static int editDistance(final String before, final String after) {
if (before == null || after == null) {
throw new IllegalArgumentException();
}
return editDistanceNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after));
}
@Override
public boolean isValidWord(final String word) {
return getFrequency(word) != NOT_A_PROBABILITY;
@ -497,22 +463,6 @@ public final class BinaryDictionary extends Dictionary {
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
/**
* Control the current time to be used in the native code. If currentTime >= 0, this method sets
* the current time and gets into test mode.
* In test mode, set timestamp is used as the current time in the native code.
* If currentTime < 0, quit the test mode and returns to using time() to get the current time.
*
* @param currentTime seconds since the unix epoch
* @return current time got in the native code.
*/
@UsedForTesting
public static int setCurrentTimeForTest(final int currentTime) {
final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime);
PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp);
return currentNativeTimestamp;
}
@UsedForTesting
public String getPropertyForTest(final String query) {
if (!isValidDictionary()) return "";

3
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.CombinedFormatUtils;
import com.android.inputmethod.latin.utils.ExecutorUtils;
@ -228,7 +229,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
private void createBinaryDictionaryLocked() {
BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
BinaryDictionaryUtils.createEmptyDictFile(mDictFile.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
}

3
java/src/com/android/inputmethod/latin/Suggest.java
View File

@ -22,6 +22,7 @@ import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.define.ProductionFlag;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.StringUtils;
@ -308,7 +309,7 @@ public final class Suggest {
// than i because we added the typed word to mSuggestions without touching mScores.
for (int i = 0; i < suggestionsSize - 1; ++i) {
final SuggestedWordInfo cur = suggestions.get(i + 1);
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
typedWord, cur.toString(), cur.mScore);
final String scoreInfoString;
if (normalizedScore > 0) {

3
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -22,6 +22,7 @@ import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import java.io.File;
@ -54,7 +55,7 @@ public class Ver4DictEncoder implements DictEncoder {
if (!mDictPlacedDir.isDirectory()) {
throw new UnsupportedFormatException("Given path is not a directory.");
}
if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
if (!BinaryDictionaryUtils.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString(
dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)),
dict.mOptions.mAttributes)) {

5
java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
View File

@ -37,6 +37,7 @@ import com.android.inputmethod.latin.SynchronouslyLoadedContactsBinaryDictionary
import com.android.inputmethod.latin.SynchronouslyLoadedUserBinaryDictionary;
import com.android.inputmethod.latin.UserBinaryDictionary;
import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import com.android.inputmethod.latin.utils.StringUtils;
@ -320,7 +321,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService
hasRecommendedSuggestions = false;
} else {
gatheredSuggestions = EMPTY_STRING_ARRAY;
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
mOriginalText, mBestSuggestion, mBestScore);
hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
}
@ -355,7 +356,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService
final int bestScore = mScores[mLength - 1];
final String bestSuggestion = mSuggestions.get(0);
final float normalizedScore =
BinaryDictionary.calcNormalizedScore(
BinaryDictionaryUtils.calcNormalizedScore(
mOriginalText, bestSuggestion.toString(), bestScore);
hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
if (DBG) {

7
java/src/com/android/inputmethod/latin/utils/AutoCorrectionUtils.java
View File

@ -40,7 +40,7 @@ public final class AutoCorrectionUtils {
final int autoCorrectionSuggestionScore = suggestion.mScore;
// TODO: when the normalized score of the first suggestion is nearly equals to
// the normalized score of the second suggestion, behave less aggressive.
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
consideredWord, suggestion.mWord, autoCorrectionSuggestionScore);
if (DBG) {
Log.d(TAG, "Normalized " + consideredWord + "," + suggestion + ","
@ -71,9 +71,8 @@ public final class AutoCorrectionUtils {
if (typedWordLength < MINIMUM_SAFETY_NET_CHAR_LENGTH) {
return false;
}
final int maxEditDistanceOfNativeDictionary =
(typedWordLength < 5 ? 2 : typedWordLength / 2) + 1;
final int distance = BinaryDictionary.editDistance(typedWord, suggestion);
final int maxEditDistanceOfNativeDictionary = (typedWordLength / 2) + 1;
final int distance = BinaryDictionaryUtils.editDistance(typedWord, suggestion);
if (DBG) {
Log.d(TAG, "Autocorrected edit distance = " + distance
+ ", " + maxEditDistanceOfNativeDictionary);

85
java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.utils;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import java.util.Locale;
import java.util.Map;
public final class BinaryDictionaryUtils {
private static final String TAG = BinaryDictionaryUtils.class.getSimpleName();
private BinaryDictionaryUtils() {
// This utility class is not publicly instantiable.
}
static {
JniUtils.loadNativeLibrary();
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
private static native int editDistanceNative(int[] before, int[] after);
private static native int setCurrentTimeForTestNative(int currentTime);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
for (final String key : attributeMap.keySet()) {
keyArray[index] = key;
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
public static float calcNormalizedScore(final String before, final String after,
final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after), score);
}
public static int editDistance(final String before, final String after) {
if (before == null || after == null) {
throw new IllegalArgumentException();
}
return editDistanceNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after));
}
/**
* Control the current time to be used in the native code. If currentTime >= 0, this method sets
* the current time and gets into test mode.
* In test mode, set timestamp is used as the current time in the native code.
* If currentTime < 0, quit the test mode and returns to using time() to get the current time.
*
* @param currentTime seconds since the unix epoch
* @return current time got in the native code.
*/
@UsedForTesting
public static int setCurrentTimeForTest(final int currentTime) {
final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime);
PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp);
return currentNativeTimestamp;
}
}

2
native/jni/com_android_inputmethod_latin_BinaryDictionaryUtils.cpp
View File

@ -130,7 +130,7 @@ static const JNINativeMethod sMethods[] = {
};
int register_BinaryDictionaryUtils(JNIEnv *env) {
const char *const kClassPathName = "com/android/inputmethod/latin/BinaryDictionary";
const char *const kClassPathName = "com/android/inputmethod/latin/utils/BinaryDictionaryUtils";
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
}
} // namespace latinime

7
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
@ -111,7 +112,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
return file;
} else {
@ -121,11 +122,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionary.setCurrentTimeForTest(currentTime);
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
}
private static int stopTestModeInNativeCode() {
return BinaryDictionary.setCurrentTimeForTest(-1);
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
public void testReadDictInJavaSide() {

3
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -25,6 +25,7 @@ import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
@ -59,7 +60,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
file.delete();
file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>();
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
Locale.ENGLISH, attributeMap)) {
return file;
} else {

7
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -30,6 +30,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
import com.android.inputmethod.latin.utils.CollectionUtils;
@ -77,7 +78,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) {
super();
BinaryDictionary.setCurrentTimeForTest(0);
BinaryDictionaryUtils.setCurrentTimeForTest(0);
Log.e(TAG, "Testing dictionary: seed is " + seed);
final Random random = new Random(seed);
sWords.clear();
@ -112,13 +113,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
@Override
protected void setUp() throws Exception {
super.setUp();
BinaryDictionary.setCurrentTimeForTest(0);
BinaryDictionaryUtils.setCurrentTimeForTest(0);
}
@Override
protected void tearDown() throws Exception {
// Quit test mode.
BinaryDictionary.setCurrentTimeForTest(-1);
BinaryDictionaryUtils.setCurrentTimeForTest(-1);
super.tearDown();
}

6
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -20,8 +20,8 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
@ -79,11 +79,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionary.setCurrentTimeForTest(currentTime);
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
}
private static int stopTestModeInNativeCode() {
return BinaryDictionary.setCurrentTimeForTest(-1);
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
/**

20
tests/src/com/android/inputmethod/latin/EditDistanceTests.java → tests/src/com/android/inputmethod/latin/utils/EditDistanceTests.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package com.android.inputmethod.latin;
package com.android.inputmethod.latin.utils;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
@ -29,7 +29,7 @@ public class EditDistanceTests extends AndroidTestCase {
* sitting
*/
public void testExample1() {
final int dist = BinaryDictionary.editDistance("kitten", "sitting");
final int dist = BinaryDictionaryUtils.editDistance("kitten", "sitting");
assertEquals("edit distance between 'kitten' and 'sitting' is 3",
3, dist);
}
@ -42,26 +42,26 @@ public class EditDistanceTests extends AndroidTestCase {
* S--unday
*/
public void testExample2() {
final int dist = BinaryDictionary.editDistance("Saturday", "Sunday");
final int dist = BinaryDictionaryUtils.editDistance("Saturday", "Sunday");
assertEquals("edit distance between 'Saturday' and 'Sunday' is 3",
3, dist);
}
public void testBothEmpty() {
final int dist = BinaryDictionary.editDistance("", "");
final int dist = BinaryDictionaryUtils.editDistance("", "");
assertEquals("when both string are empty, no edits are needed",
0, dist);
}
public void testFirstArgIsEmpty() {
final int dist = BinaryDictionary.editDistance("", "aaaa");
final int dist = BinaryDictionaryUtils.editDistance("", "aaaa");
assertEquals("when only one string of the arguments is empty,"
+ " the edit distance is the length of the other.",
4, dist);
}
public void testSecoondArgIsEmpty() {
final int dist = BinaryDictionary.editDistance("aaaa", "");
final int dist = BinaryDictionaryUtils.editDistance("aaaa", "");
assertEquals("when only one string of the arguments is empty,"
+ " the edit distance is the length of the other.",
4, dist);
@ -70,27 +70,27 @@ public class EditDistanceTests extends AndroidTestCase {
public void testSameStrings() {
final String arg1 = "The quick brown fox jumps over the lazy dog.";
final String arg2 = "The quick brown fox jumps over the lazy dog.";
final int dist = BinaryDictionary.editDistance(arg1, arg2);
final int dist = BinaryDictionaryUtils.editDistance(arg1, arg2);
assertEquals("when same strings are passed, distance equals 0.",
0, dist);
}
public void testSameReference() {
final String arg = "The quick brown fox jumps over the lazy dog.";
final int dist = BinaryDictionary.editDistance(arg, arg);
final int dist = BinaryDictionaryUtils.editDistance(arg, arg);
assertEquals("when same string references are passed, the distance equals 0.",
0, dist);
}
public void testNullArg() {
try {
BinaryDictionary.editDistance(null, "aaa");
BinaryDictionaryUtils.editDistance(null, "aaa");
fail("IllegalArgumentException should be thrown.");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
try {
BinaryDictionary.editDistance("aaa", null);
BinaryDictionaryUtils.editDistance("aaa", null);
fail("IllegalArgumentException should be thrown.");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);

1
tools/dicttool/Android.mk
View File

@ -41,6 +41,7 @@ USED_TARGETTED_UTILS := \
$(LATINIME_CORE_SOURCE_DIRECTORY)/SuggestedWords.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/WordComposer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/settings/NativeSuggestOptions.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/BinaryDictionaryUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayDictBuffer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CombinedFormatUtils.java \