Merge "[ML8] Add a language weight"

main
Jean Chalard 2014-09-19 05:15:51 +00:00 committed by Android (Google) Code Review
commit 17511a41a4
22 changed files with 109 additions and 72 deletions

View File

@ -188,7 +188,8 @@ public final class BinaryDictionary extends Dictionary {
int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray,
int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints, int prevWordCount, int[] outputSuggestionCount, int[] outputCodePoints,
int[] outputScores, int[] outputIndices, int[] outputTypes, int[] outputScores, int[] outputIndices, int[] outputTypes,
int[] outputAutoCommitFirstWordConfidence, float[] inOutLanguageWeight); int[] outputAutoCommitFirstWordConfidence,
float[] inOutWeightOfLangModelVsSpatialModel);
private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, private static native boolean addUnigramEntryNative(long dict, int[] word, int probability,
int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence,
boolean isNotAWord, boolean isBlacklisted, int timestamp); boolean isNotAWord, boolean isBlacklisted, int timestamp);
@ -256,7 +257,8 @@ public final class BinaryDictionary extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int sessionId, final float[] inOutLanguageWeight) { final int sessionId, final float weightForLocale,
final float[] inOutWeightOfLangModelVsSpatialModel) {
if (!isValidDictionary()) { if (!isValidDictionary()) {
return null; return null;
} }
@ -284,10 +286,12 @@ public final class BinaryDictionary extends Dictionary {
settingsValuesForSuggestion.mSpaceAwareGestureEnabled); settingsValuesForSuggestion.mSpaceAwareGestureEnabled);
session.mNativeSuggestOptions.setAdditionalFeaturesOptions( session.mNativeSuggestOptions.setAdditionalFeaturesOptions(
settingsValuesForSuggestion.mAdditionalFeaturesSettingValues); settingsValuesForSuggestion.mAdditionalFeaturesSettingValues);
if (inOutLanguageWeight != null) { if (inOutWeightOfLangModelVsSpatialModel != null) {
session.mInputOutputLanguageWeight[0] = inOutLanguageWeight[0]; session.mInputOutputWeightOfLangModelVsSpatialModel[0] =
inOutWeightOfLangModelVsSpatialModel[0];
} else { } else {
session.mInputOutputLanguageWeight[0] = Dictionary.NOT_A_LANGUAGE_WEIGHT; session.mInputOutputWeightOfLangModelVsSpatialModel[0] =
Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL;
} }
// TOOD: Pass multiple previous words information for n-gram. // TOOD: Pass multiple previous words information for n-gram.
getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(), getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(),
@ -298,9 +302,11 @@ public final class BinaryDictionary extends Dictionary {
session.mIsBeginningOfSentenceArray, prevWordsInfo.getPrevWordCount(), session.mIsBeginningOfSentenceArray, prevWordsInfo.getPrevWordCount(),
session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores,
session.mSpaceIndices, session.mOutputTypes, session.mSpaceIndices, session.mOutputTypes,
session.mOutputAutoCommitFirstWordConfidence, session.mInputOutputLanguageWeight); session.mOutputAutoCommitFirstWordConfidence,
if (inOutLanguageWeight != null) { session.mInputOutputWeightOfLangModelVsSpatialModel);
inOutLanguageWeight[0] = session.mInputOutputLanguageWeight[0]; if (inOutWeightOfLangModelVsSpatialModel != null) {
inOutWeightOfLangModelVsSpatialModel[0] =
session.mInputOutputWeightOfLangModelVsSpatialModel[0];
} }
final int count = session.mOutputSuggestionCount[0]; final int count = session.mOutputSuggestionCount[0];
final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>();
@ -314,7 +320,8 @@ public final class BinaryDictionary extends Dictionary {
if (len > 0) { if (len > 0) {
suggestions.add(new SuggestedWordInfo( suggestions.add(new SuggestedWordInfo(
new String(session.mOutputCodePoints, start, len), new String(session.mOutputCodePoints, start, len),
session.mOutputScores[j], session.mOutputTypes[j], this /* sourceDict */, (int)(session.mOutputScores[j] * weightForLocale), session.mOutputTypes[j],
this /* sourceDict */,
session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, session.mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */,
session.mOutputAutoCommitFirstWordConfidence[0])); session.mOutputAutoCommitFirstWordConfidence[0]));
} }

View File

@ -40,7 +40,7 @@ public final class DicTraverseSession {
public final int[] mOutputTypes = new int[MAX_RESULTS]; public final int[] mOutputTypes = new int[MAX_RESULTS];
// Only one result is ever used // Only one result is ever used
public final int[] mOutputAutoCommitFirstWordConfidence = new int[1]; public final int[] mOutputAutoCommitFirstWordConfidence = new int[1];
public final float[] mInputOutputLanguageWeight = new float[1]; public final float[] mInputOutputWeightOfLangModelVsSpatialModel = new float[1];
public final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions(); public final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions();

View File

@ -31,7 +31,7 @@ import java.util.HashSet;
*/ */
public abstract class Dictionary { public abstract class Dictionary {
public static final int NOT_A_PROBABILITY = -1; public static final int NOT_A_PROBABILITY = -1;
public static final float NOT_A_LANGUAGE_WEIGHT = -1.0f; public static final float NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL = -1.0f;
// The following types do not actually come from real dictionary instances, so we create // The following types do not actually come from real dictionary instances, so we create
// corresponding instances. // corresponding instances.
@ -88,15 +88,18 @@ public abstract class Dictionary {
* @param proximityInfo the object for key proximity. May be ignored by some implementations. * @param proximityInfo the object for key proximity. May be ignored by some implementations.
* @param settingsValuesForSuggestion the settings values used for the suggestion. * @param settingsValuesForSuggestion the settings values used for the suggestion.
* @param sessionId the session id. * @param sessionId the session id.
* @param inOutLanguageWeight the language weight used for generating suggestions. * @param weightForLocale the weight given to this locale, to multiply the output scores for
* inOutLanguageWeight is a float array that has only one element. This can be updated when the * multilingual input.
* different language weight is used. * @param inOutWeightOfLangModelVsSpatialModel the weight of the language model as a ratio of
* the spatial model, used for generating suggestions. inOutWeightOfLangModelVsSpatialModel is
* a float array that has only one element. This can be updated when a different value is used.
* @return the list of suggestions (possibly null if none) * @return the list of suggestions (possibly null if none)
*/ */
abstract public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, abstract public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int sessionId, final float[] inOutLanguageWeight); final int sessionId, final float weightForLocale,
final float[] inOutWeightOfLangModelVsSpatialModel);
/** /**
* Checks if the given word has to be treated as a valid word. Please note that some * Checks if the given word has to be treated as a valid word. Please note that some
@ -190,7 +193,8 @@ public abstract class Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int sessionId, final float[] inOutLanguageWeight) { final int sessionId, final float weightForLocale,
final float[] inOutWeightOfLangModelVsSpatialModel) {
return null; return null;
} }

View File

@ -62,20 +62,21 @@ public final class DictionaryCollection extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int sessionId, final float[] inOutLanguageWeight) { final int sessionId, final float weightForLocale,
final float[] inOutWeightOfLangModelVsSpatialModel) {
final CopyOnWriteArrayList<Dictionary> dictionaries = mDictionaries; final CopyOnWriteArrayList<Dictionary> dictionaries = mDictionaries;
if (dictionaries.isEmpty()) return null; if (dictionaries.isEmpty()) return null;
// To avoid creating unnecessary objects, we get the list out of the first // To avoid creating unnecessary objects, we get the list out of the first
// dictionary and add the rest to it if not null, hence the get(0) // dictionary and add the rest to it if not null, hence the get(0)
ArrayList<SuggestedWordInfo> suggestions = dictionaries.get(0).getSuggestions(composer, ArrayList<SuggestedWordInfo> suggestions = dictionaries.get(0).getSuggestions(composer,
prevWordsInfo, proximityInfo, settingsValuesForSuggestion, sessionId, prevWordsInfo, proximityInfo, settingsValuesForSuggestion, sessionId,
inOutLanguageWeight); weightForLocale, inOutWeightOfLangModelVsSpatialModel);
if (null == suggestions) suggestions = new ArrayList<>(); if (null == suggestions) suggestions = new ArrayList<>();
final int length = dictionaries.size(); final int length = dictionaries.size();
for (int i = 1; i < length; ++ i) { for (int i = 1; i < length; ++ i) {
final ArrayList<SuggestedWordInfo> sugg = dictionaries.get(i).getSuggestions(composer, final ArrayList<SuggestedWordInfo> sugg = dictionaries.get(i).getSuggestions(composer,
prevWordsInfo, proximityInfo, settingsValuesForSuggestion, sessionId, prevWordsInfo, proximityInfo, settingsValuesForSuggestion, sessionId,
inOutLanguageWeight); weightForLocale, inOutWeightOfLangModelVsSpatialModel);
if (null != sugg) suggestions.addAll(sugg); if (null != sugg) suggestions.addAll(sugg);
} }
return suggestions; return suggestions;

View File

@ -104,6 +104,7 @@ public class DictionaryFacilitator {
private static class DictionaryGroup { private static class DictionaryGroup {
public final Locale mLocale; public final Locale mLocale;
private Dictionary mMainDict; private Dictionary mMainDict;
public float mWeightForLocale = 1.0f;
public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap = public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap =
new ConcurrentHashMap<>(); new ConcurrentHashMap<>();
@ -598,14 +599,16 @@ public class DictionaryFacilitator {
final SuggestionResults suggestionResults = new SuggestionResults( final SuggestionResults suggestionResults = new SuggestionResults(
SuggestedWords.MAX_SUGGESTIONS, SuggestedWords.MAX_SUGGESTIONS,
prevWordsInfo.mPrevWordsInfo[0].mIsBeginningOfSentence); prevWordsInfo.mPrevWordsInfo[0].mIsBeginningOfSentence);
final float[] languageWeight = new float[] { Dictionary.NOT_A_LANGUAGE_WEIGHT }; final float[] weightOfLangModelVsSpatialModel =
new float[] { Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL };
for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { for (final DictionaryGroup dictionaryGroup : dictionaryGroups) {
for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {
final Dictionary dictionary = dictionaryGroup.getDict(dictType); final Dictionary dictionary = dictionaryGroup.getDict(dictType);
if (null == dictionary) continue; if (null == dictionary) continue;
final ArrayList<SuggestedWordInfo> dictionarySuggestions = final ArrayList<SuggestedWordInfo> dictionarySuggestions =
dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo,
settingsValuesForSuggestion, sessionId, languageWeight); settingsValuesForSuggestion, sessionId,
dictionaryGroup.mWeightForLocale, weightOfLangModelVsSpatialModel);
if (null == dictionarySuggestions) continue; if (null == dictionarySuggestions) continue;
suggestionResults.addAll(dictionarySuggestions); suggestionResults.addAll(dictionarySuggestions);
if (null != suggestionResults.mRawSuggestions) { if (null != suggestionResults.mRawSuggestions) {

View File

@ -435,7 +435,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId, final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId,
final float[] inOutLanguageWeight) { final float weightForLocale, final float[] inOutWeightOfLangModelVsSpatialModel) {
reloadDictionaryIfRequired(); reloadDictionaryIfRequired();
boolean lockAcquired = false; boolean lockAcquired = false;
try { try {
@ -447,7 +447,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
} }
final ArrayList<SuggestedWordInfo> suggestions = final ArrayList<SuggestedWordInfo> suggestions =
mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo,
settingsValuesForSuggestion, sessionId, inOutLanguageWeight); settingsValuesForSuggestion, sessionId, weightForLocale,
inOutWeightOfLangModelVsSpatialModel);
if (mBinaryDictionary.isCorrupted()) { if (mBinaryDictionary.isCorrupted()) {
Log.i(TAG, "Dictionary (" + mDictName +") is corrupted. " Log.i(TAG, "Dictionary (" + mDictName +") is corrupted. "
+ "Remove and regenerate it."); + "Remove and regenerate it.");

View File

@ -53,11 +53,13 @@ public final class ReadOnlyBinaryDictionary extends Dictionary {
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,
final SettingsValuesForSuggestion settingsValuesForSuggestion, final SettingsValuesForSuggestion settingsValuesForSuggestion,
final int sessionId, final float[] inOutLanguageWeight) { final int sessionId, final float weightForLocale,
final float[] inOutWeightOfLangModelVsSpatialModel) {
if (mLock.readLock().tryLock()) { if (mLock.readLock().tryLock()) {
try { try {
return mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, return mBinaryDictionary.getSuggestions(composer, prevWordsInfo, proximityInfo,
settingsValuesForSuggestion, sessionId, inOutLanguageWeight); settingsValuesForSuggestion, sessionId, weightForLocale,
inOutWeightOfLangModelVsSpatialModel);
} finally { } finally {
mLock.readLock().unlock(); mLock.readLock().unlock();
} }

View File

@ -182,7 +182,8 @@ static void latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz,
jobjectArray prevWordCodePointArrays, jbooleanArray isBeginningOfSentenceArray, jobjectArray prevWordCodePointArrays, jbooleanArray isBeginningOfSentenceArray,
jint prevWordCount, jintArray outSuggestionCount, jintArray outCodePointsArray, jint prevWordCount, jintArray outSuggestionCount, jintArray outCodePointsArray,
jintArray outScoresArray, jintArray outSpaceIndicesArray, jintArray outTypesArray, jintArray outScoresArray, jintArray outSpaceIndicesArray, jintArray outTypesArray,
jintArray outAutoCommitFirstWordConfidenceArray, jfloatArray inOutLanguageWeight) { jintArray outAutoCommitFirstWordConfidenceArray,
jfloatArray inOutWeightOfLangModelVsSpatialModel) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
// Assign 0 to outSuggestionCount here in case of returning earlier in this method. // Assign 0 to outSuggestionCount here in case of returning earlier in this method.
JniDataUtils::putIntToArray(env, outSuggestionCount, 0 /* index */, 0); JniDataUtils::putIntToArray(env, outSuggestionCount, 0 /* index */, 0);
@ -237,8 +238,9 @@ static void latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz,
ASSERT(false); ASSERT(false);
return; return;
} }
float languageWeight; float weightOfLangModelVsSpatialModel;
env->GetFloatArrayRegion(inOutLanguageWeight, 0, 1 /* len */, &languageWeight); env->GetFloatArrayRegion(inOutWeightOfLangModelVsSpatialModel, 0, 1 /* len */,
&weightOfLangModelVsSpatialModel);
SuggestionResults suggestionResults(MAX_RESULTS); SuggestionResults suggestionResults(MAX_RESULTS);
const PrevWordsInfo prevWordsInfo = JniDataUtils::constructPrevWordsInfo(env, const PrevWordsInfo prevWordsInfo = JniDataUtils::constructPrevWordsInfo(env,
prevWordCodePointArrays, isBeginningOfSentenceArray, prevWordCount); prevWordCodePointArrays, isBeginningOfSentenceArray, prevWordCount);
@ -246,13 +248,13 @@ static void latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz,
// TODO: Use SuggestionResults to return suggestions. // TODO: Use SuggestionResults to return suggestions.
dictionary->getSuggestions(pInfo, traverseSession, xCoordinates, yCoordinates, dictionary->getSuggestions(pInfo, traverseSession, xCoordinates, yCoordinates,
times, pointerIds, inputCodePoints, inputSize, &prevWordsInfo, times, pointerIds, inputCodePoints, inputSize, &prevWordsInfo,
&givenSuggestOptions, languageWeight, &suggestionResults); &givenSuggestOptions, weightOfLangModelVsSpatialModel, &suggestionResults);
} else { } else {
dictionary->getPredictions(&prevWordsInfo, &suggestionResults); dictionary->getPredictions(&prevWordsInfo, &suggestionResults);
} }
suggestionResults.outputSuggestions(env, outSuggestionCount, outCodePointsArray, suggestionResults.outputSuggestions(env, outSuggestionCount, outCodePointsArray,
outScoresArray, outSpaceIndicesArray, outTypesArray, outScoresArray, outSpaceIndicesArray, outTypesArray,
outAutoCommitFirstWordConfidenceArray, inOutLanguageWeight); outAutoCommitFirstWordConfidenceArray, inOutWeightOfLangModelVsSpatialModel);
} }
static jint latinime_BinaryDictionary_getProbability(JNIEnv *env, jclass clazz, jlong dict, static jint latinime_BinaryDictionary_getProbability(JNIEnv *env, jclass clazz, jlong dict,

View File

@ -301,7 +301,7 @@ static inline void prof_out(void) {
#define NOT_A_DICT_POS (S_INT_MIN) #define NOT_A_DICT_POS (S_INT_MIN)
#define NOT_A_WORD_ID (S_INT_MIN) #define NOT_A_WORD_ID (S_INT_MIN)
#define NOT_A_TIMESTAMP (-1) #define NOT_A_TIMESTAMP (-1)
#define NOT_A_LANGUAGE_WEIGHT (-1.0f) #define NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL (-1.0f)
// A special value to mean the first word confidence makes no sense in this case, // A special value to mean the first word confidence makes no sense in this case,
// e.g. this is not a multi-word suggestion. // e.g. this is not a multi-word suggestion.

View File

@ -295,8 +295,9 @@ class DicNode {
} }
// Used to prune nodes // Used to prune nodes
float getCompoundDistance(const float languageWeight) const { float getCompoundDistance(const float weightOfLangModelVsSpatialModel) const {
return mDicNodeState.mDicNodeStateScoring.getCompoundDistance(languageWeight); return mDicNodeState.mDicNodeStateScoring.getCompoundDistance(
weightOfLangModelVsSpatialModel);
} }
AK_FORCE_INLINE const int *getOutputWordBuf() const { AK_FORCE_INLINE const int *getOutputWordBuf() const {

View File

@ -103,8 +103,10 @@ class DicNodeStateScoring {
return getCompoundDistance(1.0f); return getCompoundDistance(1.0f);
} }
float getCompoundDistance(const float languageWeight) const { float getCompoundDistance(
return mSpatialDistance + mLanguageDistance * languageWeight; const float weightOfLangModelVsSpatialModel) const {
return mSpatialDistance
+ mLanguageDistance * weightOfLangModelVsSpatialModel;
} }
float getNormalizedCompoundDistance() const { float getNormalizedCompoundDistance() const {

View File

@ -47,14 +47,14 @@ Dictionary::Dictionary(JNIEnv *env, DictionaryStructureWithBufferPolicy::Structu
void Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession, void Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, const PrevWordsInfo *const prevWordsInfo, int inputSize, const PrevWordsInfo *const prevWordsInfo,
const SuggestOptions *const suggestOptions, const float languageWeight, const SuggestOptions *const suggestOptions, const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) const { SuggestionResults *const outSuggestionResults) const {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
traverseSession->init(this, prevWordsInfo, suggestOptions); traverseSession->init(this, prevWordsInfo, suggestOptions);
const auto &suggest = suggestOptions->isGesture() ? mGestureSuggest : mTypingSuggest; const auto &suggest = suggestOptions->isGesture() ? mGestureSuggest : mTypingSuggest;
suggest->getSuggestions(proximityInfo, traverseSession, xcoordinates, suggest->getSuggestions(proximityInfo, traverseSession, xcoordinates,
ycoordinates, times, pointerIds, inputCodePoints, inputSize, ycoordinates, times, pointerIds, inputCodePoints, inputSize,
languageWeight, outSuggestionResults); weightOfLangModelVsSpatialModel, outSuggestionResults);
if (DEBUG_DICT) { if (DEBUG_DICT) {
outSuggestionResults->dumpSuggestions(); outSuggestionResults->dumpSuggestions();
} }

View File

@ -66,7 +66,7 @@ class Dictionary {
void getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession, void getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, const PrevWordsInfo *const prevWordsInfo, int inputSize, const PrevWordsInfo *const prevWordsInfo,
const SuggestOptions *const suggestOptions, const float languageWeight, const SuggestOptions *const suggestOptions, const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) const; SuggestionResults *const outSuggestionResults) const;
void getPredictions(const PrevWordsInfo *const prevWordsInfo, void getPredictions(const PrevWordsInfo *const prevWordsInfo,

View File

@ -32,9 +32,11 @@ class Scoring {
const ErrorTypeUtils::ErrorType containedErrorTypes, const bool forceCommit, const ErrorTypeUtils::ErrorType containedErrorTypes, const bool forceCommit,
const bool boostExactMatches) const = 0; const bool boostExactMatches) const = 0;
virtual void getMostProbableString(const DicTraverseSession *const traverseSession, virtual void getMostProbableString(const DicTraverseSession *const traverseSession,
const float languageWeight, SuggestionResults *const outSuggestionResults) const = 0; const float weightOfLangModelVsSpatialModel,
virtual float getAdjustedLanguageWeight(DicTraverseSession *const traverseSession, SuggestionResults *const outSuggestionResults) const = 0;
DicNode *const terminals, const int size) const = 0; virtual float getAdjustedWeightOfLangModelVsSpatialModel(
DicTraverseSession *const traverseSession, DicNode *const terminals,
const int size) const = 0;
virtual float getDoubleLetterDemotionDistanceCost( virtual float getDoubleLetterDemotionDistanceCost(
const DicNode *const terminalDicNode) const = 0; const DicNode *const terminalDicNode) const = 0;
virtual bool autoCorrectsToMultiWordSuggestionIfTop() const = 0; virtual bool autoCorrectsToMultiWordSuggestionIfTop() const = 0;

View File

@ -23,7 +23,7 @@ namespace latinime {
void SuggestionResults::outputSuggestions(JNIEnv *env, jintArray outSuggestionCount, void SuggestionResults::outputSuggestions(JNIEnv *env, jintArray outSuggestionCount,
jintArray outputCodePointsArray, jintArray outScoresArray, jintArray outSpaceIndicesArray, jintArray outputCodePointsArray, jintArray outScoresArray, jintArray outSpaceIndicesArray,
jintArray outTypesArray, jintArray outAutoCommitFirstWordConfidenceArray, jintArray outTypesArray, jintArray outAutoCommitFirstWordConfidenceArray,
jfloatArray outLanguageWeight) { jfloatArray outWeightOfLangModelVsSpatialModel) {
int outputIndex = 0; int outputIndex = 0;
while (!mSuggestedWords.empty()) { while (!mSuggestedWords.empty()) {
const SuggestedWord &suggestedWord = mSuggestedWords.top(); const SuggestedWord &suggestedWord = mSuggestedWords.top();
@ -44,7 +44,8 @@ void SuggestionResults::outputSuggestions(JNIEnv *env, jintArray outSuggestionCo
mSuggestedWords.pop(); mSuggestedWords.pop();
} }
JniDataUtils::putIntToArray(env, outSuggestionCount, 0 /* index */, outputIndex); JniDataUtils::putIntToArray(env, outSuggestionCount, 0 /* index */, outputIndex);
JniDataUtils::putFloatToArray(env, outLanguageWeight, 0 /* index */, mLanguageWeight); JniDataUtils::putFloatToArray(env, outWeightOfLangModelVsSpatialModel, 0 /* index */,
mWeightOfLangModelVsSpatialModel);
} }
void SuggestionResults::addPrediction(const int *const codePoints, const int codePointCount, void SuggestionResults::addPrediction(const int *const codePoints, const int codePointCount,
@ -89,7 +90,7 @@ void SuggestionResults::getSortedScores(int *const outScores) const {
} }
void SuggestionResults::dumpSuggestions() const { void SuggestionResults::dumpSuggestions() const {
AKLOGE("language weight: %f", mLanguageWeight); AKLOGE("weight of language model vs spatial model: %f", mWeightOfLangModelVsSpatialModel);
std::vector<SuggestedWord> suggestedWords; std::vector<SuggestedWord> suggestedWords;
auto copyOfSuggestedWords = mSuggestedWords; auto copyOfSuggestedWords = mSuggestedWords;
while (!copyOfSuggestedWords.empty()) { while (!copyOfSuggestedWords.empty()) {

View File

@ -29,13 +29,15 @@ namespace latinime {
class SuggestionResults { class SuggestionResults {
public: public:
explicit SuggestionResults(const int maxSuggestionCount) explicit SuggestionResults(const int maxSuggestionCount)
: mMaxSuggestionCount(maxSuggestionCount), mLanguageWeight(NOT_A_LANGUAGE_WEIGHT), : mMaxSuggestionCount(maxSuggestionCount),
mWeightOfLangModelVsSpatialModel(NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL),
mSuggestedWords() {} mSuggestedWords() {}
// Returns suggestion count. // Returns suggestion count.
void outputSuggestions(JNIEnv *env, jintArray outSuggestionCount, jintArray outCodePointsArray, void outputSuggestions(JNIEnv *env, jintArray outSuggestionCount, jintArray outCodePointsArray,
jintArray outScoresArray, jintArray outSpaceIndicesArray, jintArray outTypesArray, jintArray outScoresArray, jintArray outSpaceIndicesArray, jintArray outTypesArray,
jintArray outAutoCommitFirstWordConfidenceArray, jfloatArray outLanguageWeight); jintArray outAutoCommitFirstWordConfidenceArray,
jfloatArray outWeightOfLangModelVsSpatialModel);
void addPrediction(const int *const codePoints, const int codePointCount, const int score); void addPrediction(const int *const codePoints, const int codePointCount, const int score);
void addSuggestion(const int *const codePoints, const int codePointCount, void addSuggestion(const int *const codePoints, const int codePointCount,
const int score, const int type, const int indexToPartialCommit, const int score, const int type, const int indexToPartialCommit,
@ -43,8 +45,8 @@ class SuggestionResults {
void getSortedScores(int *const outScores) const; void getSortedScores(int *const outScores) const;
void dumpSuggestions() const; void dumpSuggestions() const;
void setLanguageWeight(const float languageWeight) { void setWeightOfLangModelVsSpatialModel(const float weightOfLangModelVsSpatialModel) {
mLanguageWeight = languageWeight; mWeightOfLangModelVsSpatialModel = weightOfLangModelVsSpatialModel;
} }
int getSuggestionCount() const { int getSuggestionCount() const {
@ -55,7 +57,7 @@ class SuggestionResults {
DISALLOW_IMPLICIT_CONSTRUCTORS(SuggestionResults); DISALLOW_IMPLICIT_CONSTRUCTORS(SuggestionResults);
const int mMaxSuggestionCount; const int mMaxSuggestionCount;
float mLanguageWeight; float mWeightOfLangModelVsSpatialModel;
std::priority_queue< std::priority_queue<
SuggestedWord, std::vector<SuggestedWord>, SuggestedWord::Comparator> mSuggestedWords; SuggestedWord, std::vector<SuggestedWord>, SuggestedWord::Comparator> mSuggestedWords;
}; };

View File

@ -34,7 +34,8 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
/* static */ void SuggestionsOutputUtils::outputSuggestions( /* static */ void SuggestionsOutputUtils::outputSuggestions(
const Scoring *const scoringPolicy, DicTraverseSession *traverseSession, const Scoring *const scoringPolicy, DicTraverseSession *traverseSession,
const float languageWeight, SuggestionResults *const outSuggestionResults) { const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) {
#if DEBUG_EVALUATE_MOST_PROBABLE_STRING #if DEBUG_EVALUATE_MOST_PROBABLE_STRING
const int terminalSize = 0; const int terminalSize = 0;
#else #else
@ -44,12 +45,15 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
for (int index = terminalSize - 1; index >= 0; --index) { for (int index = terminalSize - 1; index >= 0; --index) {
traverseSession->getDicTraverseCache()->popTerminal(&terminals[index]); traverseSession->getDicTraverseCache()->popTerminal(&terminals[index]);
} }
// Compute a language weight when an invalid language weight is passed. // Compute a weight of language model when an invalid weight is passed.
// NOT_A_LANGUAGE_WEIGHT (-1) is assumed as an invalid language weight. // NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL (-1) is taken as an invalid value.
const float languageWeightToOutputSuggestions = (languageWeight < 0.0f) ? const float weightOfLangModelVsSpatialModelToOutputSuggestions =
scoringPolicy->getAdjustedLanguageWeight( (weightOfLangModelVsSpatialModel < 0.0f)
traverseSession, terminals.data(), terminalSize) : languageWeight; ? scoringPolicy->getAdjustedWeightOfLangModelVsSpatialModel(traverseSession,
outSuggestionResults->setLanguageWeight(languageWeightToOutputSuggestions); terminals.data(), terminalSize)
: weightOfLangModelVsSpatialModel;
outSuggestionResults->setWeightOfLangModelVsSpatialModel(
weightOfLangModelVsSpatialModelToOutputSuggestions);
// Force autocorrection for obvious long multi-word suggestions when the top suggestion is // Force autocorrection for obvious long multi-word suggestions when the top suggestion is
// a long multiple words suggestion. // a long multiple words suggestion.
// TODO: Implement a smarter auto-commit method for handling multi-word suggestions. // TODO: Implement a smarter auto-commit method for handling multi-word suggestions.
@ -65,16 +69,16 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
// Output suggestion results here // Output suggestion results here
for (auto &terminalDicNode : terminals) { for (auto &terminalDicNode : terminals) {
outputSuggestionsOfDicNode(scoringPolicy, traverseSession, &terminalDicNode, outputSuggestionsOfDicNode(scoringPolicy, traverseSession, &terminalDicNode,
languageWeightToOutputSuggestions, boostExactMatches, forceCommitMultiWords, weightOfLangModelVsSpatialModelToOutputSuggestions, boostExactMatches,
outputSecondWordFirstLetterInputIndex, outSuggestionResults); forceCommitMultiWords, outputSecondWordFirstLetterInputIndex, outSuggestionResults);
} }
scoringPolicy->getMostProbableString(traverseSession, languageWeightToOutputSuggestions, scoringPolicy->getMostProbableString(traverseSession,
outSuggestionResults); weightOfLangModelVsSpatialModelToOutputSuggestions, outSuggestionResults);
} }
/* static */ void SuggestionsOutputUtils::outputSuggestionsOfDicNode( /* static */ void SuggestionsOutputUtils::outputSuggestionsOfDicNode(
const Scoring *const scoringPolicy, DicTraverseSession *traverseSession, const Scoring *const scoringPolicy, DicTraverseSession *traverseSession,
const DicNode *const terminalDicNode, const float languageWeight, const DicNode *const terminalDicNode, const float weightOfLangModelVsSpatialModel,
const bool boostExactMatches, const bool forceCommitMultiWords, const bool boostExactMatches, const bool forceCommitMultiWords,
const bool outputSecondWordFirstLetterInputIndex, const bool outputSecondWordFirstLetterInputIndex,
SuggestionResults *const outSuggestionResults) { SuggestionResults *const outSuggestionResults) {
@ -83,8 +87,9 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
} }
const float doubleLetterCost = const float doubleLetterCost =
scoringPolicy->getDoubleLetterDemotionDistanceCost(terminalDicNode); scoringPolicy->getDoubleLetterDemotionDistanceCost(terminalDicNode);
const float compoundDistance = terminalDicNode->getCompoundDistance(languageWeight) const float compoundDistance =
+ doubleLetterCost; terminalDicNode->getCompoundDistance(weightOfLangModelVsSpatialModel)
+ doubleLetterCost;
const WordAttributes wordAttributes = traverseSession->getDictionaryStructurePolicy() const WordAttributes wordAttributes = traverseSession->getDictionaryStructurePolicy()
->getWordAttributesInContext(terminalDicNode->getPrevWordIds(), ->getWordAttributesInContext(terminalDicNode->getPrevWordIds(),
terminalDicNode->getWordId(), nullptr /* multiBigramMap */); terminalDicNode->getWordId(), nullptr /* multiBigramMap */);

View File

@ -33,7 +33,7 @@ class SuggestionsOutputUtils {
* Outputs the final list of suggestions (i.e., terminal nodes). * Outputs the final list of suggestions (i.e., terminal nodes).
*/ */
static void outputSuggestions(const Scoring *const scoringPolicy, static void outputSuggestions(const Scoring *const scoringPolicy,
DicTraverseSession *traverseSession, const float languageWeight, DicTraverseSession *traverseSession, const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults); SuggestionResults *const outSuggestionResults);
private: private:
@ -44,7 +44,7 @@ class SuggestionsOutputUtils {
static void outputSuggestionsOfDicNode(const Scoring *const scoringPolicy, static void outputSuggestionsOfDicNode(const Scoring *const scoringPolicy,
DicTraverseSession *traverseSession, const DicNode *const terminalDicNode, DicTraverseSession *traverseSession, const DicNode *const terminalDicNode,
const float languageWeight, const bool boostExactMatches, const float weightOfLangModelVsSpatialModel, const bool boostExactMatches,
const bool forceCommitMultiWords, const bool outputSecondWordFirstLetterInputIndex, const bool forceCommitMultiWords, const bool outputSecondWordFirstLetterInputIndex,
SuggestionResults *const outSuggestionResults); SuggestionResults *const outSuggestionResults);
static void outputShortcuts(BinaryDictionaryShortcutIterator *const shortcutIt, static void outputShortcuts(BinaryDictionaryShortcutIterator *const shortcutIt,

View File

@ -45,7 +45,7 @@ const int Suggest::MIN_CONTINUOUS_SUGGESTION_INPUT_SIZE = 2;
*/ */
void Suggest::getSuggestions(ProximityInfo *pInfo, void *traverseSession, void Suggest::getSuggestions(ProximityInfo *pInfo, void *traverseSession,
int *inputXs, int *inputYs, int *times, int *pointerIds, int *inputCodePoints, int *inputXs, int *inputYs, int *times, int *pointerIds, int *inputCodePoints,
int inputSize, const float languageWeight, int inputSize, const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) const { SuggestionResults *const outSuggestionResults) const {
PROF_OPEN; PROF_OPEN;
PROF_START(0); PROF_START(0);
@ -68,7 +68,7 @@ void Suggest::getSuggestions(ProximityInfo *pInfo, void *traverseSession,
PROF_END(1); PROF_END(1);
PROF_START(2); PROF_START(2);
SuggestionsOutputUtils::outputSuggestions( SuggestionsOutputUtils::outputSuggestions(
SCORING, tSession, languageWeight, outSuggestionResults); SCORING, tSession, weightOfLangModelVsSpatialModel, outSuggestionResults);
PROF_END(2); PROF_END(2);
PROF_CLOSE; PROF_CLOSE;
} }

View File

@ -49,7 +49,8 @@ class Suggest : public SuggestInterface {
AK_FORCE_INLINE virtual ~Suggest() {} AK_FORCE_INLINE virtual ~Suggest() {}
void getSuggestions(ProximityInfo *pInfo, void *traverseSession, int *inputXs, int *inputYs, void getSuggestions(ProximityInfo *pInfo, void *traverseSession, int *inputXs, int *inputYs,
int *times, int *pointerIds, int *inputCodePoints, int inputSize, int *times, int *pointerIds, int *inputCodePoints, int inputSize,
const float languageWeight, SuggestionResults *const outSuggestionResults) const; const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) const;
private: private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Suggest); DISALLOW_IMPLICIT_CONSTRUCTORS(Suggest);

View File

@ -28,7 +28,8 @@ class SuggestInterface {
public: public:
virtual void getSuggestions(ProximityInfo *pInfo, void *traverseSession, int *inputXs, virtual void getSuggestions(ProximityInfo *pInfo, void *traverseSession, int *inputXs,
int *inputYs, int *times, int *pointerIds, int *inputCodePoints, int inputSize, int *inputYs, int *times, int *pointerIds, int *inputCodePoints, int inputSize,
const float languageWeight, SuggestionResults *const suggestionResults) const = 0; const float weightOfLangModelVsSpatialModel,
SuggestionResults *const suggestionResults) const = 0;
SuggestInterface() {} SuggestInterface() {}
virtual ~SuggestInterface() {} virtual ~SuggestInterface() {}
private: private:

View File

@ -33,10 +33,12 @@ class TypingScoring : public Scoring {
static const TypingScoring *getInstance() { return &sInstance; } static const TypingScoring *getInstance() { return &sInstance; }
AK_FORCE_INLINE void getMostProbableString(const DicTraverseSession *const traverseSession, AK_FORCE_INLINE void getMostProbableString(const DicTraverseSession *const traverseSession,
const float languageWeight, SuggestionResults *const outSuggestionResults) const {} const float weightOfLangModelVsSpatialModel,
SuggestionResults *const outSuggestionResults) const {}
AK_FORCE_INLINE float getAdjustedLanguageWeight(DicTraverseSession *const traverseSession, AK_FORCE_INLINE float getAdjustedWeightOfLangModelVsSpatialModel(
DicNode *const terminals, const int size) const { DicTraverseSession *const traverseSession, DicNode *const terminals,
const int size) const {
return 1.0f; return 1.0f;
} }