nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Differentiate exact matches' minor errors."

main
Keisuke Kuroyanagi 2013-12-19 04:14:11 +00:00 committed by Android (Google) Code Review
parent abe3a35abf afa6c07d30
commit b68dd6cd0b
8 changed files with 55 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
native/jni/src/suggest/core/dicnode/dic_node.h
View File

@ -23,6 +23,7 @@
#include "suggest/core/dicnode/internal/dic_node_state.h"
#include "suggest/core/dicnode/internal/dic_node_properties.h"
#include "suggest/core/dictionary/digraph_utils.h"
#include "suggest/core/dictionary/error_type_utils.h"
#include "utils/char_utils.h"
#if DEBUG_DICT
@ -493,8 +494,8 @@ class DicNode {
mDicNodeState.mDicNodeStateScoring.advanceDigraphIndex();
}
bool isExactMatch() const {
return mDicNodeState.mDicNodeStateScoring.isExactMatch();
ErrorTypeUtils::ErrorType getContainedErrorTypes() const {
return mDicNodeState.mDicNodeStateScoring.getContainedErrorTypes();
}
bool isBlacklistedOrNotAWord() const {
@ -535,8 +536,8 @@ class DicNode {
return false;
}
// Promote exact matches to prevent them from being pruned.
const bool leftExactMatch = isExactMatch();
const bool rightExactMatch = right->isExactMatch();
const bool leftExactMatch = ErrorTypeUtils::isExactMatch(getContainedErrorTypes());
const bool rightExactMatch = ErrorTypeUtils::isExactMatch(right->getContainedErrorTypes());
if (leftExactMatch != rightExactMatch) {
return leftExactMatch;
}

14
native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h
View File

@ -32,7 +32,7 @@ class DicNodeStateScoring {
mDigraphIndex(DigraphUtils::NOT_A_DIGRAPH_INDEX),
mEditCorrectionCount(0), mProximityCorrectionCount(0),
mNormalizedCompoundDistance(0.0f), mSpatialDistance(0.0f), mLanguageDistance(0.0f),
mRawLength(0.0f), mContainingErrorTypes(ErrorTypeUtils::NOT_AN_ERROR),
mRawLength(0.0f), mContainedErrorTypes(ErrorTypeUtils::NOT_AN_ERROR),
mNormalizedCompoundDistanceAfterFirstWord(MAX_VALUE_FOR_WEIGHTING) {
}
@ -48,7 +48,7 @@ class DicNodeStateScoring {
mDoubleLetterLevel = NOT_A_DOUBLE_LETTER;
mDigraphIndex = DigraphUtils::NOT_A_DIGRAPH_INDEX;
mNormalizedCompoundDistanceAfterFirstWord = MAX_VALUE_FOR_WEIGHTING;
mContainingErrorTypes = ErrorTypeUtils::NOT_AN_ERROR;
mContainedErrorTypes = ErrorTypeUtils::NOT_AN_ERROR;
}
AK_FORCE_INLINE void init(const DicNodeStateScoring *const scoring) {
@ -60,7 +60,7 @@ class DicNodeStateScoring {
mRawLength = scoring->mRawLength;
mDoubleLetterLevel = scoring->mDoubleLetterLevel;
mDigraphIndex = scoring->mDigraphIndex;
mContainingErrorTypes = scoring->mContainingErrorTypes;
mContainedErrorTypes = scoring->mContainedErrorTypes;
mNormalizedCompoundDistanceAfterFirstWord =
scoring->mNormalizedCompoundDistanceAfterFirstWord;
}
@ -69,7 +69,7 @@ class DicNodeStateScoring {
const int inputSize, const int totalInputIndex,
const ErrorTypeUtils::ErrorType errorType) {
addDistance(spatialCost, languageCost, doNormalization, inputSize, totalInputIndex);
mContainingErrorTypes = mContainingErrorTypes | errorType;
mContainedErrorTypes = mContainedErrorTypes | errorType;
if (ErrorTypeUtils::isEditCorrectionError(errorType)) {
++mEditCorrectionCount;
}
@ -169,8 +169,8 @@ class DicNodeStateScoring {
}
}
bool isExactMatch() const {
return ErrorTypeUtils::isExactMatch(mContainingErrorTypes);
ErrorTypeUtils::ErrorType getContainedErrorTypes() const {
return mContainedErrorTypes;
}
private:
@ -188,7 +188,7 @@ class DicNodeStateScoring {
float mLanguageDistance;
float mRawLength;
// All accumulated error types so far
ErrorTypeUtils::ErrorType mContainingErrorTypes;
ErrorTypeUtils::ErrorType mContainedErrorTypes;
float mNormalizedCompoundDistanceAfterFirstWord;
AK_FORCE_INLINE void addDistance(float spatialDistance, float languageDistance,

5
native/jni/src/suggest/core/dictionary/error_type_utils.h
View File

@ -47,9 +47,8 @@ class ErrorTypeUtils {
// A new word error should be an edit correction error or a proximity correction error.
static const ErrorType NEW_WORD;
// TODO: Differentiate errors.
static bool isExactMatch(const ErrorType containingErrors) {
return (containingErrors & ~ERRORS_TREATED_AS_AN_EXACT_MATCH) == 0;
static bool isExactMatch(const ErrorType containedErrorTypes) {
return (containedErrorTypes & ~ERRORS_TREATED_AS_AN_EXACT_MATCH) == 0;
}
static bool isEditCorrectionError(const ErrorType errorType) {

16
native/jni/src/suggest/core/dictionary/suggestions_output_utils.cpp
View File

@ -18,8 +18,9 @@
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_utils.h"
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/dictionary/binary_dictionary_shortcut_iterator.h"
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/dictionary/error_type_utils.h"
#include "suggest/core/policy/scoring.h"
#include "suggest/core/session/dic_traverse_session.h"
@ -98,7 +99,8 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
const bool isPossiblyOffensiveWord =
traverseSession->getDictionaryStructurePolicy()->getProbability(
terminalDicNode->getProbability(), NOT_A_PROBABILITY) <= 0;
const bool isExactMatch = terminalDicNode->isExactMatch();
const bool isExactMatch =
ErrorTypeUtils::isExactMatch(terminalDicNode->getContainedErrorTypes());
const bool isFirstCharUppercase = terminalDicNode->isFirstCharUppercase();
// Heuristic: We exclude freq=0 first-char-uppercase words from exact match.
// (e.g. "AMD" and "and")
@ -115,9 +117,9 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
// TODO: Better integration with java side autocorrection logic.
const int finalScore = scoringPolicy->calculateFinalScore(
compoundDistance, traverseSession->getInputSize(),
terminalDicNode->isExactMatch()
|| (forceCommitMultiWords && terminalDicNode->hasMultipleWords())
|| (isValidWord && scoringPolicy->doesAutoCorrectValidWord()));
terminalDicNode->getContainedErrorTypes(),
(forceCommitMultiWords && terminalDicNode->hasMultipleWords())
|| (isValidWord && scoringPolicy->doesAutoCorrectValidWord()));
if (maxScore < finalScore && isValidWord) {
maxScore = finalScore;
}
@ -149,7 +151,9 @@ const int SuggestionsOutputUtils::MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT = 16;
const bool sameAsTyped = scoringPolicy->sameAsTyped(traverseSession, terminalDicNode);
const int shortcutBaseScore = scoringPolicy->doesAutoCorrectValidWord() ?
scoringPolicy->calculateFinalScore(compoundDistance,
traverseSession->getInputSize(), true /* forceCommit */) : finalScore;
traverseSession->getInputSize(),
terminalDicNode->getContainedErrorTypes(),
true /* forceCommit */) : finalScore;
const int updatedOutputWordIndex = outputShortcuts(&shortcutIt,
outputWordIndex, shortcutBaseScore, outputCodePoints, frequencies, outputTypes,
sameAsTyped);

2
native/jni/src/suggest/core/policy/scoring.h
View File

@ -28,7 +28,7 @@ class DicTraverseSession;
class Scoring {
public:
virtual int calculateFinalScore(const float compoundDistance, const int inputSize,
const bool forceCommit) const = 0;
const ErrorTypeUtils::ErrorType containedErrorTypes, const bool forceCommit) const = 0;
virtual bool getMostProbableString(const DicTraverseSession *const traverseSession,
const int terminalSize, const float languageWeight, int *const outputCodePoints,
int *const type, int *const freq) const = 0;

6
native/jni/src/suggest/policyimpl/typing/scoring_params.cpp
View File

@ -22,6 +22,12 @@ const float ScoringParams::MAX_SPATIAL_DISTANCE = 1.0f;
const int ScoringParams::THRESHOLD_NEXT_WORD_PROBABILITY = 40;
const int ScoringParams::THRESHOLD_NEXT_WORD_PROBABILITY_FOR_CAPPED = 120;
const float ScoringParams::AUTOCORRECT_OUTPUT_THRESHOLD = 1.0f;
const float ScoringParams::EXACT_MATCH_PROMOTION = 1.1f;
const float ScoringParams::CASE_ERROR_PENALTY_FOR_EXACT_MATCH = 0.01f;
const float ScoringParams::ACCENT_ERROR_PENALTY_FOR_EXACT_MATCH = 0.02f;
const float ScoringParams::DIGRAPH_PENALTY_FOR_EXACT_MATCH = 0.03f;
// TODO: Unlimit max cache dic node size
const int ScoringParams::MAX_CACHE_DIC_NODE_SIZE = 170;
const int ScoringParams::MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT = 310;

5
native/jni/src/suggest/policyimpl/typing/scoring_params.h
View File

@ -32,6 +32,11 @@ class ScoringParams {
static const int MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT;
static const int THRESHOLD_SHORT_WORD_LENGTH;
static const float EXACT_MATCH_PROMOTION;
static const float CASE_ERROR_PENALTY_FOR_EXACT_MATCH;
static const float ACCENT_ERROR_PENALTY_FOR_EXACT_MATCH;
static const float DIGRAPH_PENALTY_FOR_EXACT_MATCH;
// Numerically optimized parameters (currently for tap typing only).
// TODO: add ability to modify these constants programmatically.
// TODO: explore optimization of gesture parameters.

23
native/jni/src/suggest/policyimpl/typing/typing_scoring.h
View File

@ -18,6 +18,7 @@
#define LATINIME_TYPING_SCORING_H
#include "defines.h"
#include "suggest/core/dictionary/error_type_utils.h"
#include "suggest/core/policy/scoring.h"
#include "suggest/core/session/dic_traverse_session.h"
#include "suggest/policyimpl/typing/scoring_params.h"
@ -53,12 +54,26 @@ class TypingScoring : public Scoring {
}
AK_FORCE_INLINE int calculateFinalScore(const float compoundDistance,
const int inputSize, const bool forceCommit) const {
const int inputSize, const ErrorTypeUtils::ErrorType containedErrorTypes,
const bool forceCommit) const {
const float maxDistance = ScoringParams::DISTANCE_WEIGHT_LANGUAGE
+ static_cast<float>(inputSize) * ScoringParams::TYPING_MAX_OUTPUT_SCORE_PER_INPUT;
const float score = ScoringParams::TYPING_BASE_OUTPUT_SCORE
- compoundDistance / maxDistance
+ (forceCommit ? ScoringParams::AUTOCORRECT_OUTPUT_THRESHOLD : 0.0f);
float score = ScoringParams::TYPING_BASE_OUTPUT_SCORE - compoundDistance / maxDistance;
if (forceCommit) {
score += ScoringParams::AUTOCORRECT_OUTPUT_THRESHOLD;
}
if (ErrorTypeUtils::isExactMatch(containedErrorTypes)) {
score += ScoringParams::EXACT_MATCH_PROMOTION;
if ((ErrorTypeUtils::MATCH_WITH_CASE_ERROR & containedErrorTypes) != 0) {
score -= ScoringParams::CASE_ERROR_PENALTY_FOR_EXACT_MATCH;
}
if ((ErrorTypeUtils::MATCH_WITH_ACCENT_ERROR & containedErrorTypes) != 0) {
score -= ScoringParams::ACCENT_ERROR_PENALTY_FOR_EXACT_MATCH;
}
if ((ErrorTypeUtils::MATCH_WITH_DIGRAPH & containedErrorTypes) != 0) {
score -= ScoringParams::DIGRAPH_PENALTY_FOR_EXACT_MATCH;
}
}
return static_cast<int>(score * SUGGEST_INTERFACE_OUTPUT_SCALE);
}