nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Separate utility methods from BinaryDictionary."

main
Keisuke Kuroyanagi 2014-03-05 10:07:33 +00:00 committed by Android (Google) Code Review
parent 4f6ce0df0b e784148ae6
commit 274eae95ba
14 changed files with 122 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -40,7 +40,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
/**
* Implements a static, compacted, binary dictionary of standard words.
@ -142,8 +141,6 @@ public final class BinaryDictionary extends Dictionary {
JniUtils.loadNativeLibrary();
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
@ -167,8 +164,6 @@ public final class BinaryDictionary extends Dictionary {
int[] suggestOptions, int[] prevWordCodePointArray,
int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes,
int[] outputAutoCommitFirstWordConfidence);
private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
private static native int editDistanceNative(int[] before, int[] after);
private static native void addUnigramWordNative(long dict, int[] word, int probability,
int[] shortcutTarget, int shortcutProbability, boolean isNotAWord,
boolean isBlacklisted, int timestamp);
@ -179,24 +174,9 @@ public final class BinaryDictionary extends Dictionary {
LanguageModelParam[] languageModelParams, int startIndex);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
private static native int setCurrentTimeForTestNative(int currentTime);
private static native String getPropertyNative(long dict, String query);
private static native boolean isCorruptedNative(long dict);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
for (final String key : attributeMap.keySet()) {
keyArray[index] = key;
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
// TODO: Move native dict into session
private final void loadDictionary(final String path, final long startOffset,
final long length, final boolean isUpdatable) {
@ -323,20 +303,6 @@ public final class BinaryDictionary extends Dictionary {
return getFormatVersionNative(mNativeDict);
}
public static float calcNormalizedScore(final String before, final String after,
final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after), score);
}
public static int editDistance(final String before, final String after) {
if (before == null || after == null) {
throw new IllegalArgumentException();
}
return editDistanceNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after));
}
@Override
public boolean isValidWord(final String word) {
return getFrequency(word) != NOT_A_PROBABILITY;
@ -497,22 +463,6 @@ public final class BinaryDictionary extends Dictionary {
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
/**
* Control the current time to be used in the native code. If currentTime >= 0, this method sets
* the current time and gets into test mode.
* In test mode, set timestamp is used as the current time in the native code.
* If currentTime < 0, quit the test mode and returns to using time() to get the current time.
*
* @param currentTime seconds since the unix epoch
* @return current time got in the native code.
*/
@UsedForTesting
public static int setCurrentTimeForTest(final int currentTime) {
final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime);
PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp);
return currentNativeTimestamp;
}
@UsedForTesting
public String getPropertyForTest(final String query) {
if (!isValidDictionary()) return "";

3
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.CombinedFormatUtils;
import com.android.inputmethod.latin.utils.ExecutorUtils;
@ -228,7 +229,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
private void createBinaryDictionaryLocked() {
BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(),
BinaryDictionaryUtils.createEmptyDictFile(mDictFile.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
}

3
java/src/com/android/inputmethod/latin/Suggest.java
View File

@ -22,6 +22,7 @@ import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.define.ProductionFlag;
import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.BoundedTreeSet;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.StringUtils;
@ -309,7 +310,7 @@ public final class Suggest {
// than i because we added the typed word to mSuggestions without touching mScores.
for (int i = 0; i < suggestionsSize - 1; ++i) {
final SuggestedWordInfo cur = suggestions.get(i + 1);
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
typedWord, cur.toString(), cur.mScore);
final String scoreInfoString;
if (normalizedScore > 0) {

3
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -22,6 +22,7 @@ import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import java.io.File;
@ -54,7 +55,7 @@ public class Ver4DictEncoder implements DictEncoder {
if (!mDictPlacedDir.isDirectory()) {
throw new UnsupportedFormatException("Given path is not a directory.");
}
if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
if (!BinaryDictionaryUtils.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString(
dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)),
dict.mOptions.mAttributes)) {

5
java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
View File

@ -37,6 +37,7 @@ import com.android.inputmethod.latin.SynchronouslyLoadedContactsBinaryDictionary
import com.android.inputmethod.latin.SynchronouslyLoadedUserBinaryDictionary;
import com.android.inputmethod.latin.UserBinaryDictionary;
import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import com.android.inputmethod.latin.utils.StringUtils;
@ -320,7 +321,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService
hasRecommendedSuggestions = false;
} else {
gatheredSuggestions = EMPTY_STRING_ARRAY;
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
mOriginalText, mBestSuggestion, mBestScore);
hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
}
@ -355,7 +356,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService
final int bestScore = mScores[mLength - 1];
final String bestSuggestion = mSuggestions.get(0);
final float normalizedScore =
BinaryDictionary.calcNormalizedScore(
BinaryDictionaryUtils.calcNormalizedScore(
mOriginalText, bestSuggestion.toString(), bestScore);
hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
if (DBG) {

7
java/src/com/android/inputmethod/latin/utils/AutoCorrectionUtils.java
View File

@ -40,7 +40,7 @@ public final class AutoCorrectionUtils {
final int autoCorrectionSuggestionScore = suggestion.mScore;
// TODO: when the normalized score of the first suggestion is nearly equals to
// the normalized score of the second suggestion, behave less aggressive.
final float normalizedScore = BinaryDictionary.calcNormalizedScore(
final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
consideredWord, suggestion.mWord, autoCorrectionSuggestionScore);
if (DBG) {
Log.d(TAG, "Normalized " + consideredWord + "," + suggestion + ","
@ -71,9 +71,8 @@ public final class AutoCorrectionUtils {
if (typedWordLength < MINIMUM_SAFETY_NET_CHAR_LENGTH) {
return false;
}
final int maxEditDistanceOfNativeDictionary =
(typedWordLength < 5 ? 2 : typedWordLength / 2) + 1;
final int distance = BinaryDictionary.editDistance(typedWord, suggestion);
final int maxEditDistanceOfNativeDictionary = (typedWordLength / 2) + 1;
final int distance = BinaryDictionaryUtils.editDistance(typedWord, suggestion);
if (DBG) {
Log.d(TAG, "Autocorrected edit distance = " + distance
+ ", " + maxEditDistanceOfNativeDictionary);

85
java/src/com/android/inputmethod/latin/utils/BinaryDictionaryUtils.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.utils;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.personalization.PersonalizationHelper;
import java.util.Locale;
import java.util.Map;
public final class BinaryDictionaryUtils {
private static final String TAG = BinaryDictionaryUtils.class.getSimpleName();
private BinaryDictionaryUtils() {
// This utility class is not publicly instantiable.
}
static {
JniUtils.loadNativeLibrary();
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native float calcNormalizedScoreNative(int[] before, int[] after, int score);
private static native int editDistanceNative(int[] before, int[] after);
private static native int setCurrentTimeForTestNative(int currentTime);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
for (final String key : attributeMap.keySet()) {
keyArray[index] = key;
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
public static float calcNormalizedScore(final String before, final String after,
final int score) {
return calcNormalizedScoreNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after), score);
}
public static int editDistance(final String before, final String after) {
if (before == null || after == null) {
throw new IllegalArgumentException();
}
return editDistanceNative(StringUtils.toCodePointArray(before),
StringUtils.toCodePointArray(after));
}
/**
* Control the current time to be used in the native code. If currentTime >= 0, this method sets
* the current time and gets into test mode.
* In test mode, set timestamp is used as the current time in the native code.
* If currentTime < 0, quit the test mode and returns to using time() to get the current time.
*
* @param currentTime seconds since the unix epoch
* @return current time got in the native code.
*/
@UsedForTesting
public static int setCurrentTimeForTest(final int currentTime) {
final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime);
PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp);
return currentNativeTimestamp;
}
}

2
native/jni/com_android_inputmethod_latin_BinaryDictionaryUtils.cpp
View File

@ -130,7 +130,7 @@ static const JNINativeMethod sMethods[] = {
};
int register_BinaryDictionaryUtils(JNIEnv *env) {
const char *const kClassPathName = "com/android/inputmethod/latin/BinaryDictionary";
const char *const kClassPathName = "com/android/inputmethod/latin/utils/BinaryDictionaryUtils";
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
}
} // namespace latinime

7
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -27,6 +27,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
@ -111,7 +112,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
return file;
} else {
@ -121,11 +122,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionary.setCurrentTimeForTest(currentTime);
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
}
private static int stopTestModeInNativeCode() {
return BinaryDictionary.setCurrentTimeForTest(-1);
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
public void testReadDictInJavaSide() {

3
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -25,6 +25,7 @@ import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
@ -59,7 +60,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
file.delete();
file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>();
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
Locale.ENGLISH, attributeMap)) {
return file;
} else {

7
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -30,6 +30,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
import com.android.inputmethod.latin.utils.CollectionUtils;
@ -77,7 +78,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) {
super();
BinaryDictionary.setCurrentTimeForTest(0);
BinaryDictionaryUtils.setCurrentTimeForTest(0);
Log.e(TAG, "Testing dictionary: seed is " + seed);
final Random random = new Random(seed);
sWords.clear();
@ -112,13 +113,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
@Override
protected void setUp() throws Exception {
super.setUp();
BinaryDictionary.setCurrentTimeForTest(0);
BinaryDictionaryUtils.setCurrentTimeForTest(0);
}
@Override
protected void tearDown() throws Exception {
// Quit test mode.
BinaryDictionary.setCurrentTimeForTest(-1);
BinaryDictionaryUtils.setCurrentTimeForTest(-1);
super.tearDown();
}

6
tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
View File

@ -20,8 +20,8 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
@ -79,11 +79,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
}
private static int setCurrentTimeForTestMode(final int currentTime) {
return BinaryDictionary.setCurrentTimeForTest(currentTime);
return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime);
}
private static int stopTestModeInNativeCode() {
return BinaryDictionary.setCurrentTimeForTest(-1);
return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
}
/**

20
tests/src/com/android/inputmethod/latin/EditDistanceTests.java → tests/src/com/android/inputmethod/latin/utils/EditDistanceTests.java
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
package com.android.inputmethod.latin;
package com.android.inputmethod.latin.utils;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
@ -29,7 +29,7 @@ public class EditDistanceTests extends AndroidTestCase {
* sitting
*/
public void testExample1() {
final int dist = BinaryDictionary.editDistance("kitten", "sitting");
final int dist = BinaryDictionaryUtils.editDistance("kitten", "sitting");
assertEquals("edit distance between 'kitten' and 'sitting' is 3",
3, dist);
}
@ -42,26 +42,26 @@ public class EditDistanceTests extends AndroidTestCase {
* S--unday
*/
public void testExample2() {
final int dist = BinaryDictionary.editDistance("Saturday", "Sunday");
final int dist = BinaryDictionaryUtils.editDistance("Saturday", "Sunday");
assertEquals("edit distance between 'Saturday' and 'Sunday' is 3",
3, dist);
}
public void testBothEmpty() {
final int dist = BinaryDictionary.editDistance("", "");
final int dist = BinaryDictionaryUtils.editDistance("", "");
assertEquals("when both string are empty, no edits are needed",
0, dist);
}
public void testFirstArgIsEmpty() {
final int dist = BinaryDictionary.editDistance("", "aaaa");
final int dist = BinaryDictionaryUtils.editDistance("", "aaaa");
assertEquals("when only one string of the arguments is empty,"
+ " the edit distance is the length of the other.",
4, dist);
}
public void testSecoondArgIsEmpty() {
final int dist = BinaryDictionary.editDistance("aaaa", "");
final int dist = BinaryDictionaryUtils.editDistance("aaaa", "");
assertEquals("when only one string of the arguments is empty,"
+ " the edit distance is the length of the other.",
4, dist);
@ -70,27 +70,27 @@ public class EditDistanceTests extends AndroidTestCase {
public void testSameStrings() {
final String arg1 = "The quick brown fox jumps over the lazy dog.";
final String arg2 = "The quick brown fox jumps over the lazy dog.";
final int dist = BinaryDictionary.editDistance(arg1, arg2);
final int dist = BinaryDictionaryUtils.editDistance(arg1, arg2);
assertEquals("when same strings are passed, distance equals 0.",
0, dist);
}
public void testSameReference() {
final String arg = "The quick brown fox jumps over the lazy dog.";
final int dist = BinaryDictionary.editDistance(arg, arg);
final int dist = BinaryDictionaryUtils.editDistance(arg, arg);
assertEquals("when same string references are passed, the distance equals 0.",
0, dist);
}
public void testNullArg() {
try {
BinaryDictionary.editDistance(null, "aaa");
BinaryDictionaryUtils.editDistance(null, "aaa");
fail("IllegalArgumentException should be thrown.");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
}
try {
BinaryDictionary.editDistance("aaa", null);
BinaryDictionaryUtils.editDistance("aaa", null);
fail("IllegalArgumentException should be thrown.");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);

1
tools/dicttool/Android.mk
View File

@ -41,6 +41,7 @@ USED_TARGETTED_UTILS := \
$(LATINIME_CORE_SOURCE_DIRECTORY)/SuggestedWords.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/WordComposer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/settings/NativeSuggestOptions.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/BinaryDictionaryUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ByteArrayDictBuffer.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CollectionUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/CombinedFormatUtils.java \