nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 01c1a1a2: Refactor LanguageModelParam class a bit 

* commit '01c1a1a2ba4a75ae3e52196a45e13fee598c5b09':
  Refactor LanguageModelParam class a bit
main
Tadashi G. Takaoka 2014-01-15 01:40:25 -08:00 committed by Android Git Automerger
parent 55df3234b2 01c1a1a2ba
commit 15ef522203
1 changed files with 113 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java
View File

@ -16,9 +16,29 @@
package com.android.inputmethod.latin.utils;
import android.util.Log;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest;
import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
import java.util.ArrayList;
import java.util.Locale;
public class LanguageModelParam {
private static final String TAG = LanguageModelParam.class.getSimpleName();
private static final boolean DEBUG = false;
private static final boolean DEBUG_TOKEN = false;
// For now, these probability values are being referred to only when we add new entries to
// decaying dynamic binary dictionaries. When these are referred to, what matters is 0 or
// non-0. Thus, it's not meaningful to compare 10, 100, and so on.
// TODO: Revise the logic in ForgettingCurveUtils in native code.
private static final int UNIGRAM_PROBABILITY_FOR_VALID_WORD = 100;
private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = 10;
private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 0;
private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = 0;
public final String mTargetWord;
public final int[] mWord0;
public final int[] mWord1;
@ -53,4 +73,97 @@ public class LanguageModelParam {
mIsBlacklisted = false;
mTimestamp = timestamp;
}
// Process a list of words and return a list of {@link LanguageModelParam} objects.
public static ArrayList<LanguageModelParam> createLanguageModelParamsFrom(
final ArrayList<String> tokens, final int timestamp,
final DictionaryFacilitatorForSuggest dictionaryFacilitator,
final SpacingAndPunctuations spacingAndPunctuations) {
final ArrayList<LanguageModelParam> languageModelParams =
CollectionUtils.newArrayList();
final int N = tokens.size();
String prevWord = null;
for (int i = 0; i < N; ++i) {
final String tempWord = tokens.get(i);
if (StringUtils.isEmptyStringOrWhiteSpaces(tempWord)) {
// just skip this token
if (DEBUG_TOKEN) {
Log.d(TAG, "--- isEmptyStringOrWhiteSpaces: \"" + tempWord + "\"");
}
continue;
}
if (!DictionaryInfoUtils.looksValidForDictionaryInsertion(
tempWord, spacingAndPunctuations)) {
if (DEBUG_TOKEN) {
Log.d(TAG, "--- not looksValidForDictionaryInsertion: \""
+ tempWord + "\"");
}
// Sentence terminator found. Split.
prevWord = null;
continue;
}
if (DEBUG_TOKEN) {
Log.d(TAG, "--- word: \"" + tempWord + "\"");
}
final LanguageModelParam languageModelParam =
detectWhetherVaildWordOrNotAndGetLanguageModelParam(
prevWord, tempWord, timestamp, dictionaryFacilitator);
languageModelParams.add(languageModelParam);
prevWord = languageModelParam.mTargetWord;
}
return languageModelParams;
}
private static LanguageModelParam detectWhetherVaildWordOrNotAndGetLanguageModelParam(
final String prevWord, final String targetWord, final int timestamp,
final DictionaryFacilitatorForSuggest dictionaryFacilitator) {
final Locale locale = dictionaryFacilitator.mLocale;
if (!dictionaryFacilitator.isValidWord(targetWord, true /* ignoreCase */)) {
// OOV word.
return createAndGetLanguageModelParamOfWord(prevWord, targetWord, timestamp,
false /* isValidWord */, locale);
}
if (dictionaryFacilitator.isValidWord(targetWord, false /* ignoreCase */)) {
return createAndGetLanguageModelParamOfWord(prevWord, targetWord, timestamp,
true /* isValidWord */, locale);
}
final String lowerCaseTargetWord = targetWord.toLowerCase(locale);
if (dictionaryFacilitator.isValidWord(lowerCaseTargetWord, false /* ignoreCase */)) {
// Add the lower-cased word.
return createAndGetLanguageModelParamOfWord(prevWord, lowerCaseTargetWord,
timestamp, true /* isValidWord */, locale);
}
// Treat the word as an OOV word.
return createAndGetLanguageModelParamOfWord(prevWord, targetWord, timestamp,
false /* isValidWord */, locale);
}
private static LanguageModelParam createAndGetLanguageModelParamOfWord(
final String prevWord, final String targetWord, final int timestamp,
final boolean isValidWord, final Locale locale) {
final String word;
if (StringUtils.getCapitalizationType(targetWord) == StringUtils.CAPITALIZE_FIRST
&& prevWord == null && !isValidWord) {
word = targetWord.toLowerCase(locale);
} else {
word = targetWord;
}
final int unigramProbability = isValidWord ?
UNIGRAM_PROBABILITY_FOR_VALID_WORD : UNIGRAM_PROBABILITY_FOR_OOV_WORD;
if (prevWord == null) {
if (DEBUG) {
Log.d(TAG, "--- add unigram: current("
+ (isValidWord ? "Valid" : "OOV") + ") = " + word);
}
return new LanguageModelParam(word, unigramProbability, timestamp);
}
if (DEBUG) {
Log.d(TAG, "--- add bigram: prev = " + prevWord + ", current("
+ (isValidWord ? "Valid" : "OOV") + ") = " + word);
}
final int bigramProbability = isValidWord ?
BIGRAM_PROBABILITY_FOR_VALID_WORD : BIGRAM_PROBABILITY_FOR_OOV_WORD;
return new LanguageModelParam(prevWord, word, unigramProbability,
bigramProbability, timestamp);
}
}