nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Store raw strings for personal dictionary"

main
Jatin Matani 2015-04-14 18:03:55 +00:00 committed by Android (Google) Code Review
parent db96122787 1bfd7be208
commit bf732e7001
2 changed files with 67 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
java/src/com/android/inputmethod/latin/PersonalDictionaryLookup.java
View File

@ -40,9 +40,9 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -196,11 +196,10 @@ public class PersonalDictionaryLookup implements Closeable {
private AtomicBoolean mIsClosed = new AtomicBoolean(false); private AtomicBoolean mIsClosed = new AtomicBoolean(false);
/** /**
* We store a map from a dictionary word to the set of locales it belongs * We store a map from a dictionary word to the set of locales & raw string(as it appears)
* in. We then iterate over the set of locales to find a match using * We then iterate over the set of locales to find a match using LocaleUtils.
* LocaleUtils.
*/ */
private volatile HashMap<String, ArrayList<Locale>> mDictWords; private volatile HashMap<String, HashMap<Locale, String>> mDictWords;
/** /**
* We store a map from a shortcut to a word for each locale. * We store a map from a shortcut to a word for each locale.
@ -317,7 +316,7 @@ public class PersonalDictionaryLookup implements Closeable {
* @return set of words that apply to the given locale. * @return set of words that apply to the given locale.
*/ */
public Set<String> getWordsForLocale(@Nonnull final Locale inputLocale) { public Set<String> getWordsForLocale(@Nonnull final Locale inputLocale) {
final HashMap<String, ArrayList<Locale>> dictWords = mDictWords; final HashMap<String, HashMap<Locale, String>> dictWords = mDictWords;
if (CollectionUtils.isNullOrEmpty(dictWords)) { if (CollectionUtils.isNullOrEmpty(dictWords)) {
return Collections.emptySet(); return Collections.emptySet();
} }
@ -325,12 +324,15 @@ public class PersonalDictionaryLookup implements Closeable {
final Set<String> words = new HashSet<>(); final Set<String> words = new HashSet<>();
final String inputLocaleString = inputLocale.toString(); final String inputLocaleString = inputLocale.toString();
for (String word : dictWords.keySet()) { for (String word : dictWords.keySet()) {
for (Locale wordLocale : dictWords.get(word)) { HashMap<Locale, String> localeStringMap = dictWords.get(word);
final String wordLocaleString = wordLocale.toString(); if (!CollectionUtils.isNullOrEmpty(localeStringMap)) {
final int match = LocaleUtils.getMatchLevel(wordLocaleString, inputLocaleString); for (Locale wordLocale : localeStringMap.keySet()) {
if (LocaleUtils.isMatch(match)) { final String wordLocaleString = wordLocale.toString();
words.add(word); final int match = LocaleUtils.getMatchLevel(wordLocaleString, inputLocaleString);
} if (LocaleUtils.isMatch(match)) {
words.add(localeStringMap.get(wordLocale));
}
}
} }
} }
return words; return words;
@ -399,29 +401,29 @@ public class PersonalDictionaryLookup implements Closeable {
return false; return false;
} }
// Atomically obtain the current copy of mDictWords;
final HashMap<String, ArrayList<Locale>> dictWords = mDictWords;
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
Log.d(mTag, "isValidWord() : Word [" + word + "] in Locale [" + inputLocale + "]"); Log.d(mTag, "isValidWord() : Word [" + word + "] in Locale [" + inputLocale + "]");
} }
// Atomically obtain the current copy of mDictWords;
final HashMap<String, HashMap<Locale, String>> dictWords = mDictWords;
// Lowercase the word using the given locale. Note, that dictionary // Lowercase the word using the given locale. Note, that dictionary
// words are lowercased using their locale, and theoretically the // words are lowercased using their locale, and theoretically the
// lowercasing between two matching locales may differ. For simplicity // lowercasing between two matching locales may differ. For simplicity
// we ignore that possibility. // we ignore that possibility.
final String lowercased = word.toLowerCase(inputLocale); final String lowercased = word.toLowerCase(inputLocale);
final ArrayList<Locale> dictLocales = dictWords.get(lowercased); final HashMap<Locale, String> dictLocales = dictWords.get(lowercased);
if (null == dictLocales) {
if (CollectionUtils.isNullOrEmpty(dictLocales)) {
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
Log.d(mTag, "isValidWord() : No entry for lowercased word [" + lowercased + "]"); Log.d(mTag, "isValidWord() : No entry for word [" + word + "]");
} }
return false; return false;
} else { } else {
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
Log.d(mTag, "isValidWord() : Found entry for lowercased word [" + lowercased + "]"); Log.d(mTag, "isValidWord() : Found entry for word [" + word + "]");
} }
// Iterate over the locales this word is in. // Iterate over the locales this word is in.
for (final Locale dictLocale : dictLocales) { for (final Locale dictLocale : dictLocales.keySet()) {
final int matchLevel = LocaleUtils.getMatchLevel(dictLocale.toString(), final int matchLevel = LocaleUtils.getMatchLevel(dictLocale.toString(),
inputLocale.toString()); inputLocale.toString());
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
@ -529,7 +531,7 @@ public class PersonalDictionaryLookup implements Closeable {
return; return;
} }
Log.i(mTag, "loadPersonalDictionary() : Start Loading"); Log.i(mTag, "loadPersonalDictionary() : Start Loading");
HashMap<String, ArrayList<Locale>> dictWords = new HashMap<>(); HashMap<String, HashMap<Locale, String>> dictWords = new HashMap<>();
HashMap<Locale, HashMap<String, String>> shortcutsPerLocale = new HashMap<>(); HashMap<Locale, HashMap<String, String>> shortcutsPerLocale = new HashMap<>();
// Load the dictionary. Items are returned in the default sort order (by frequency). // Load the dictionary. Items are returned in the default sort order (by frequency).
Cursor cursor = mResolver.query(UserDictionary.Words.CONTENT_URI, Cursor cursor = mResolver.query(UserDictionary.Words.CONTENT_URI,
@ -581,21 +583,21 @@ public class PersonalDictionaryLookup implements Closeable {
final String dictWord = rawDictWord.toLowerCase(dictLocale); final String dictWord = rawDictWord.toLowerCase(dictLocale);
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
Log.d(mTag, "loadPersonalDictionary() : Adding word [" + dictWord Log.d(mTag, "loadPersonalDictionary() : Adding word [" + dictWord
+ "] for locale " + dictLocale); + "] for locale " + dictLocale + "with value" + rawDictWord);
} }
// Check if there is an existing entry for this word. // Check if there is an existing entry for this word.
ArrayList<Locale> dictLocales = dictWords.get(dictWord); HashMap<Locale, String> dictLocales = dictWords.get(dictWord);
if (null == dictLocales) { if (CollectionUtils.isNullOrEmpty(dictLocales)) {
// If there is no entry for this word, create one. // If there is no entry for this word, create one.
if (DebugFlags.DEBUG_ENABLED) { if (DebugFlags.DEBUG_ENABLED) {
Log.d(mTag, "loadPersonalDictionary() : Word [" + dictWord + Log.d(mTag, "loadPersonalDictionary() : Word [" + dictWord +
"] not seen for other locales, creating new entry"); "] not seen for other locales, creating new entry");
} }
dictLocales = new ArrayList<>(); dictLocales = new HashMap<>();
dictWords.put(dictWord, dictLocales); dictWords.put(dictWord, dictLocales);
} }
// Append the locale to the list of locales this word is in. // Append the locale to the list of locales this word is in.
dictLocales.add(dictLocale); dictLocales.put(dictLocale, rawDictWord);
// If there is no column for a shortcut, we're done. // If there is no column for a shortcut, we're done.
final int shortcutIndex = cursor.getColumnIndex(UserDictionary.Words.SHORTCUT); final int shortcutIndex = cursor.getColumnIndex(UserDictionary.Words.SHORTCUT);

40
tests/src/com/android/inputmethod/latin/PersonalDictionaryLookupTest.java
View File

@ -289,7 +289,8 @@ public class PersonalDictionaryLookupTest extends AndroidTestCase {
addWord("fOo", Locale.FRENCH, 17, null); addWord("fOo", Locale.FRENCH, 17, null);
// Create the PersonalDictionaryLookup and wait until it's loaded. // Create the PersonalDictionaryLookup and wait until it's loaded.
PersonalDictionaryLookup lookup = new PersonalDictionaryLookup(mContext, ExecutorUtils.SPELLING); PersonalDictionaryLookup lookup = new PersonalDictionaryLookup(mContext,
ExecutorUtils.SPELLING);
lookup.open(); lookup.open();
// Both en_CA and en_US match. // Both en_CA and en_US match.
@ -304,6 +305,43 @@ public class PersonalDictionaryLookupTest extends AndroidTestCase {
lookup.close(); lookup.close();
} }
public void testCaseMatchingForWordsAndShortcuts() {
Log.d(TAG, "testCaseMatchingForWordsAndShortcuts");
addWord("Foo", Locale.US, 17, "f");
addWord("bokabu", Locale.US, 17, "Bu");
// Create the PersonalDictionaryLookup and wait until it's loaded.
PersonalDictionaryLookup lookup = new PersonalDictionaryLookup(mContext,
ExecutorUtils.SPELLING);
lookup.open();
// Valid, inspite of capitalization in US but not in other
// locales.
assertTrue(lookup.isValidWord("Foo", Locale.US));
assertTrue(lookup.isValidWord("foo", Locale.US));
assertFalse(lookup.isValidWord("Foo", Locale.UK));
assertFalse(lookup.isValidWord("foo", Locale.UK));
// Valid in all forms in US.
assertTrue(lookup.isValidWord("bokabu", Locale.US));
assertTrue(lookup.isValidWord("BOKABU", Locale.US));
assertTrue(lookup.isValidWord("BokaBU", Locale.US));
// Correct capitalization; sensitive to shortcut casing & locale.
assertEquals("Foo", lookup.expandShortcut("f", Locale.US));
assertNull(lookup.expandShortcut("f", Locale.UK));
// Correct capitalization; sensitive to shortcut casing & locale.
assertEquals("bokabu", lookup.expandShortcut("Bu", Locale.US));
assertNull(lookup.expandShortcut("Bu", Locale.UK));
assertNull(lookup.expandShortcut("bu", Locale.US));
// Verify that raw strings are retained for #getWordsForLocale.
verifyWordExists(lookup.getWordsForLocale(Locale.US), "Foo");
verifyWordDoesNotExist(lookup.getWordsForLocale(Locale.US), "foo");
}
public void testManageListeners() { public void testManageListeners() {
Log.d(TAG, "testManageListeners"); Log.d(TAG, "testManageListeners");