nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove unused methods. 

Bug: 8550444

Change-Id: Ie627084143846407ca36df82c2a7f904a0e6f49c
main
Keisuke Kuroynagi 2013-06-11 17:04:40 +09:00
parent cd4edc15a5
commit 527c128309
17 changed files with 185 additions and 1662 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
native/jni/Android.mk
View File

@ -47,8 +47,6 @@ LATIN_IME_JNI_SRC_FILES := \
LATIN_IME_CORE_SRC_FILES := \
suggest/core/suggest.cpp \
$(addprefix obsolete/, \
correction.cpp) \
$(addprefix suggest/core/dicnode/, \
dic_node.cpp \
dic_node_utils.cpp \
@ -76,7 +74,9 @@ LATIN_IME_CORE_SRC_FILES := \
typing_suggest_policy.cpp \
typing_traversal.cpp \
typing_weighting.cpp) \
utils/char_utils.cpp
$(addprefix utils/, \
char_utils.cpp \
ranking_algorithm.cpp)
LOCAL_SRC_FILES := \
$(LATIN_IME_JNI_SRC_FILES) \

7
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -22,15 +22,16 @@
#include <cstring> // for memset()
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include "defines.h"
#include "jni.h"
#include "jni_common.h"
#include "obsolete/correction.h"
#include "suggest/core/dictionary/binary_dictionary_format_utils.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/suggest_options.h"
#include "utils/autocorrection_threshold_utils.h"
namespace latinime {
@ -201,7 +202,7 @@ static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jclass
int afterCodePoints[afterLength];
env->GetIntArrayRegion(before, 0, beforeLength, beforeCodePoints);
env->GetIntArrayRegion(after, 0, afterLength, afterCodePoints);
return Correction::RankingAlgorithm::calcNormalizedScore(beforeCodePoints, beforeLength,
return AutocorrectionThresholdUtils::calcNormalizedScore(beforeCodePoints, beforeLength,
afterCodePoints, afterLength, score);
}
@ -213,7 +214,7 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji
int afterCodePoints[afterLength];
env->GetIntArrayRegion(before, 0, beforeLength, beforeCodePoints);
env->GetIntArrayRegion(after, 0, afterLength, afterCodePoints);
return Correction::RankingAlgorithm::editDistance(beforeCodePoints, beforeLength,
return AutocorrectionThresholdUtils::editDistance(beforeCodePoints, beforeLength,
afterCodePoints, afterLength);
}

1006
native/jni/src/obsolete/correction.cpp
View File

File diff suppressed because it is too large Load Diff

398
native/jni/src/obsolete/correction.h
View File

@ -1,398 +0,0 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_CORRECTION_H
#define LATINIME_CORRECTION_H
#include <cstring> // for memset()
#include "defines.h"
#include "obsolete/correction_state.h"
#include "suggest/core/layout/proximity_info_state.h"
#include "utils/char_utils.h"
namespace latinime {
class ProximityInfo;
class Correction {
public:
typedef enum {
TRAVERSE_ALL_ON_TERMINAL,
TRAVERSE_ALL_NOT_ON_TERMINAL,
UNRELATED,
ON_TERMINAL,
NOT_ON_TERMINAL
} CorrectionType;
Correction()
: mProximityInfo(0), mUseFullEditDistance(false), mDoAutoCompletion(false),
mMaxEditDistance(0), mMaxDepth(0), mInputSize(0), mSpaceProximityPos(0),
mMissingSpacePos(0), mTerminalInputIndex(0), mTerminalOutputIndex(0), mMaxErrors(0),
mTotalTraverseCount(0), mNeedsToTraverseAllNodes(false), mOutputIndex(0),
mInputIndex(0), mEquivalentCharCount(0), mProximityCount(0), mExcessiveCount(0),
mTransposedCount(0), mSkippedCount(0), mTransposedPos(0), mExcessivePos(0),
mSkipPos(0), mLastCharExceeded(false), mMatching(false), mProximityMatching(false),
mAdditionalProximityMatching(false), mExceeding(false), mTransposing(false),
mSkipping(false), mProximityInfoState() {
memset(mWord, 0, sizeof(mWord));
memset(mDistances, 0, sizeof(mDistances));
memset(mEditDistanceTable, 0, sizeof(mEditDistanceTable));
// NOTE: mCorrectionStates is an array of instances.
// No need to initialize it explicitly here.
}
// Non virtual inline destructor -- never inherit this class
~Correction() {}
void resetCorrection();
void initCorrection(const ProximityInfo *pi, const int inputSize, const int maxDepth);
void initCorrectionState(const int rootPos, const int childCount, const bool traverseAll);
// TODO: remove
void setCorrectionParams(const int skipPos, const int excessivePos, const int transposedPos,
const int spaceProximityPos, const int missingSpacePos, const bool useFullEditDistance,
const bool doAutoCompletion, const int maxErrors);
void checkState() const;
bool sameAsTyped() const;
bool initProcessState(const int index);
int getInputIndex() const;
bool needsToPrune() const;
int pushAndGetTotalTraverseCount() {
return ++mTotalTraverseCount;
}
int getFreqForSplitMultipleWords(const int *freqArray, const int *wordLengthArray,
const int wordCount, const bool isSpaceProximity, const int *word) const;
int getFinalProbability(const int probability, int **word, int *wordLength);
int getFinalProbabilityForSubQueue(const int probability, int **word, int *wordLength,
const int inputSize);
CorrectionType processCharAndCalcState(const int c, const bool isTerminal);
/////////////////////////
// Tree helper methods
int goDownTree(const int parentIndex, const int childCount, const int firstChildPos);
inline int getTreeSiblingPos(const int index) const {
return mCorrectionStates[index].mSiblingPos;
}
inline void setTreeSiblingPos(const int index, const int pos) {
mCorrectionStates[index].mSiblingPos = pos;
}
inline int getTreeParentIndex(const int index) const {
return mCorrectionStates[index].mParentIndex;
}
class RankingAlgorithm {
public:
static int calculateFinalProbability(const int inputIndex, const int depth,
const int probability, int *editDistanceTable, const Correction *correction,
const int inputSize);
static int calcFreqForSplitMultipleWords(const int *freqArray, const int *wordLengthArray,
const int wordCount, const Correction *correction, const bool isSpaceProximity,
const int *word);
static float calcNormalizedScore(const int *before, const int beforeLength,
const int *after, const int afterLength, const int score);
static int editDistance(const int *before, const int beforeLength, const int *after,
const int afterLength);
private:
static const int MAX_INITIAL_SCORE = 255;
};
// proximity info state
void initInputParams(const ProximityInfo *proximityInfo, const int *inputCodes,
const int inputSize, const int *xCoordinates, const int *yCoordinates) {
mProximityInfoState.initInputParams(0, static_cast<float>(MAX_VALUE_FOR_WEIGHTING),
proximityInfo, inputCodes, inputSize, xCoordinates, yCoordinates, 0, 0, false);
}
const int *getPrimaryInputWord() const {
return mProximityInfoState.getPrimaryInputWord();
}
int getPrimaryCodePointAt(const int index) const {
return mProximityInfoState.getPrimaryCodePointAt(index);
}
private:
DISALLOW_COPY_AND_ASSIGN(Correction);
// The following "rate"s are used as a multiplier before dividing by 100, so they are in
// percent.
static const int WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE;
static const int WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X;
static const int WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE;
static const int WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE;
static const int WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE;
static const int WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE;
static const int WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE;
static const int FULL_MATCHED_WORDS_PROMOTION_RATE;
static const int WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE;
static const int WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE;
static const int WORDS_WITH_MATCH_SKIP_PROMOTION_RATE;
static const int WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE;
static const int WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER;
static const int CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE;
static const int INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE;
static const int FIRST_CHAR_DIFFERENT_DEMOTION_RATE;
static const int TWO_WORDS_CAPITALIZED_DEMOTION_RATE;
static const int TWO_WORDS_CORRECTION_DEMOTION_BASE;
/////////////////////////
// static inline utils //
/////////////////////////
static const int TWO_31ST_DIV_255 = S_INT_MAX / 255;
static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) {
return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX);
}
static const int TWO_31ST_DIV_2 = S_INT_MAX / 2;
AK_FORCE_INLINE static void multiplyIntCapped(const int multiplier, int *base) {
const int temp = *base;
if (temp != S_INT_MAX) {
// Branch if multiplier == 2 for the optimization
if (multiplier < 0) {
if (DEBUG_DICT) {
ASSERT(false);
}
AKLOGI("--- Invalid multiplier: %d", multiplier);
} else if (multiplier == 0) {
*base = 0;
} else if (multiplier == 2) {
*base = TWO_31ST_DIV_2 >= temp ? temp << 1 : S_INT_MAX;
} else {
// TODO: This overflow check gives a wrong answer when, for example,
// temp = 2^16 + 1 and multiplier = 2^17 + 1.
// Fix this behavior.
const int tempRetval = temp * multiplier;
*base = tempRetval >= temp ? tempRetval : S_INT_MAX;
}
}
}
AK_FORCE_INLINE static int powerIntCapped(const int base, const int n) {
if (n <= 0) return 1;
if (base == 2) {
return n < 31 ? 1 << n : S_INT_MAX;
}
int ret = base;
for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret);
return ret;
}
AK_FORCE_INLINE static void multiplyRate(const int rate, int *freq) {
if (*freq != S_INT_MAX) {
if (*freq > 1000000) {
*freq /= 100;
multiplyIntCapped(rate, freq);
} else {
multiplyIntCapped(rate, freq);
*freq /= 100;
}
}
}
inline int getSpaceProximityPos() const {
return mSpaceProximityPos;
}
inline int getMissingSpacePos() const {
return mMissingSpacePos;
}
inline int getSkipPos() const {
return mSkipPos;
}
inline int getExcessivePos() const {
return mExcessivePos;
}
inline int getTransposedPos() const {
return mTransposedPos;
}
inline void incrementInputIndex();
inline void incrementOutputIndex();
inline void startToTraverseAllNodes();
inline bool isSingleQuote(const int c);
inline CorrectionType processSkipChar(const int c, const bool isTerminal,
const bool inputIndexIncremented);
inline CorrectionType processUnrelatedCorrectionType();
inline void addCharToCurrentWord(const int c);
inline int getFinalProbabilityInternal(const int probability, int **word, int *wordLength,
const int inputSize);
static const int TYPED_LETTER_MULTIPLIER = 2;
static const int FULL_WORD_MULTIPLIER = 2;
const ProximityInfo *mProximityInfo;
bool mUseFullEditDistance;
bool mDoAutoCompletion;
int mMaxEditDistance;
int mMaxDepth;
int mInputSize;
int mSpaceProximityPos;
int mMissingSpacePos;
int mTerminalInputIndex;
int mTerminalOutputIndex;
int mMaxErrors;
int mTotalTraverseCount;
// The following arrays are state buffer.
int mWord[MAX_WORD_LENGTH];
int mDistances[MAX_WORD_LENGTH];
// Edit distance calculation requires a buffer with (N+1)^2 length for the input length N.
// Caveat: Do not create multiple tables per thread as this table eats up RAM a lot.
int mEditDistanceTable[(MAX_WORD_LENGTH + 1) * (MAX_WORD_LENGTH + 1)];
CorrectionState mCorrectionStates[MAX_WORD_LENGTH];
// The following member variables are being used as cache values of the correction state.
bool mNeedsToTraverseAllNodes;
int mOutputIndex;
int mInputIndex;
int mEquivalentCharCount;
int mProximityCount;
int mExcessiveCount;
int mTransposedCount;
int mSkippedCount;
int mTransposedPos;
int mExcessivePos;
int mSkipPos;
bool mLastCharExceeded;
bool mMatching;
bool mProximityMatching;
bool mAdditionalProximityMatching;
bool mExceeding;
bool mTransposing;
bool mSkipping;
ProximityInfoState mProximityInfoState;
};
inline void Correction::incrementInputIndex() {
++mInputIndex;
}
AK_FORCE_INLINE void Correction::incrementOutputIndex() {
++mOutputIndex;
mCorrectionStates[mOutputIndex].mParentIndex = mCorrectionStates[mOutputIndex - 1].mParentIndex;
mCorrectionStates[mOutputIndex].mChildCount = mCorrectionStates[mOutputIndex - 1].mChildCount;
mCorrectionStates[mOutputIndex].mSiblingPos = mCorrectionStates[mOutputIndex - 1].mSiblingPos;
mCorrectionStates[mOutputIndex].mInputIndex = mInputIndex;
mCorrectionStates[mOutputIndex].mNeedsToTraverseAllNodes = mNeedsToTraverseAllNodes;
mCorrectionStates[mOutputIndex].mEquivalentCharCount = mEquivalentCharCount;
mCorrectionStates[mOutputIndex].mProximityCount = mProximityCount;
mCorrectionStates[mOutputIndex].mTransposedCount = mTransposedCount;
mCorrectionStates[mOutputIndex].mExcessiveCount = mExcessiveCount;
mCorrectionStates[mOutputIndex].mSkippedCount = mSkippedCount;
mCorrectionStates[mOutputIndex].mSkipPos = mSkipPos;
mCorrectionStates[mOutputIndex].mTransposedPos = mTransposedPos;
mCorrectionStates[mOutputIndex].mExcessivePos = mExcessivePos;
mCorrectionStates[mOutputIndex].mLastCharExceeded = mLastCharExceeded;
mCorrectionStates[mOutputIndex].mMatching = mMatching;
mCorrectionStates[mOutputIndex].mProximityMatching = mProximityMatching;
mCorrectionStates[mOutputIndex].mAdditionalProximityMatching = mAdditionalProximityMatching;
mCorrectionStates[mOutputIndex].mTransposing = mTransposing;
mCorrectionStates[mOutputIndex].mExceeding = mExceeding;
mCorrectionStates[mOutputIndex].mSkipping = mSkipping;
}
inline void Correction::startToTraverseAllNodes() {
mNeedsToTraverseAllNodes = true;
}
AK_FORCE_INLINE bool Correction::isSingleQuote(const int c) {
const int userTypedChar = mProximityInfoState.getPrimaryCodePointAt(mInputIndex);
return (c == KEYCODE_SINGLE_QUOTE && userTypedChar != KEYCODE_SINGLE_QUOTE);
}
AK_FORCE_INLINE Correction::CorrectionType Correction::processSkipChar(const int c,
const bool isTerminal, const bool inputIndexIncremented) {
addCharToCurrentWord(c);
mTerminalInputIndex = mInputIndex - (inputIndexIncremented ? 1 : 0);
mTerminalOutputIndex = mOutputIndex;
incrementOutputIndex();
if (mNeedsToTraverseAllNodes && isTerminal) {
return TRAVERSE_ALL_ON_TERMINAL;
}
return TRAVERSE_ALL_NOT_ON_TERMINAL;
}
inline Correction::CorrectionType Correction::processUnrelatedCorrectionType() {
// Needs to set mTerminalInputIndex and mTerminalOutputIndex before returning any CorrectionType
mTerminalInputIndex = mInputIndex;
mTerminalOutputIndex = mOutputIndex;
return UNRELATED;
}
AK_FORCE_INLINE static void calcEditDistanceOneStep(int *editDistanceTable, const int *input,
const int inputSize, const int *output, const int outputLength) {
// TODO: Make sure that editDistance[0 ~ MAX_WORD_LENGTH] is not touched.
// Let dp[i][j] be editDistanceTable[i * (inputSize + 1) + j].
// Assuming that dp[0][0] ... dp[outputLength - 1][inputSize] are already calculated,
// and calculate dp[ouputLength][0] ... dp[outputLength][inputSize].
int *const current = editDistanceTable + outputLength * (inputSize + 1);
const int *const prev = editDistanceTable + (outputLength - 1) * (inputSize + 1);
const int *const prevprev =
outputLength >= 2 ? editDistanceTable + (outputLength - 2) * (inputSize + 1) : 0;
current[0] = outputLength;
const int co = CharUtils::toBaseLowerCase(output[outputLength - 1]);
const int prevCO = outputLength >= 2 ? CharUtils::toBaseLowerCase(output[outputLength - 2]) : 0;
for (int i = 1; i <= inputSize; ++i) {
const int ci = CharUtils::toBaseLowerCase(input[i - 1]);
const int cost = (ci == co) ? 0 : 1;
current[i] = min(current[i - 1] + 1, min(prev[i] + 1, prev[i - 1] + cost));
if (i >= 2 && prevprev && ci == prevCO && co == CharUtils::toBaseLowerCase(input[i - 2])) {
current[i] = min(current[i], prevprev[i - 2] + 1);
}
}
}
AK_FORCE_INLINE void Correction::addCharToCurrentWord(const int c) {
mWord[mOutputIndex] = c;
const int *primaryInputWord = mProximityInfoState.getPrimaryInputWord();
calcEditDistanceOneStep(mEditDistanceTable, primaryInputWord, mInputSize, mWord,
mOutputIndex + 1);
}
inline int Correction::getFinalProbabilityInternal(const int probability, int **word,
int *wordLength, const int inputSize) {
const int outputIndex = mTerminalOutputIndex;
const int inputIndex = mTerminalInputIndex;
*wordLength = outputIndex + 1;
*word = mWord;
int finalProbability= Correction::RankingAlgorithm::calculateFinalProbability(
inputIndex, outputIndex, probability, mEditDistanceTable, this, inputSize);
return finalProbability;
}
} // namespace latinime
#endif // LATINIME_CORRECTION_H

83
native/jni/src/obsolete/correction_state.h
View File

@ -1,83 +0,0 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_CORRECTION_STATE_H
#define LATINIME_CORRECTION_STATE_H
#include <stdint.h>
#include "defines.h"
namespace latinime {
struct CorrectionState {
int mParentIndex;
int mSiblingPos;
uint16_t mChildCount;
uint8_t mInputIndex;
uint8_t mEquivalentCharCount;
uint8_t mProximityCount;
uint8_t mTransposedCount;
uint8_t mExcessiveCount;
uint8_t mSkippedCount;
int8_t mTransposedPos;
int8_t mExcessivePos;
int8_t mSkipPos; // should be signed
// TODO: int?
bool mLastCharExceeded;
bool mMatching;
bool mTransposing;
bool mExceeding;
bool mSkipping;
bool mProximityMatching;
bool mAdditionalProximityMatching;
bool mNeedsToTraverseAllNodes;
};
inline static void initCorrectionState(CorrectionState *state, const int rootPos,
const uint16_t childCount, const bool traverseAll) {
state->mParentIndex = -1;
state->mChildCount = childCount;
state->mInputIndex = 0;
state->mSiblingPos = rootPos;
state->mNeedsToTraverseAllNodes = traverseAll;
state->mTransposedPos = -1;
state->mExcessivePos = -1;
state->mSkipPos = -1;
state->mEquivalentCharCount = 0;
state->mProximityCount = 0;
state->mTransposedCount = 0;
state->mExcessiveCount = 0;
state->mSkippedCount = 0;
state->mLastCharExceeded = false;
state->mMatching = false;
state->mProximityMatching = false;
state->mTransposing = false;
state->mExceeding = false;
state->mSkipping = false;
state->mAdditionalProximityMatching = false;
}
} // namespace latinime
#endif // LATINIME_CORRECTION_STATE_H

25
native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp
View File

@ -23,7 +23,6 @@
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/binary_format.h"
#include "suggest/core/dictionary/bloom_filter.h"
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/dictionary/probability_utils.h"
#include "utils/char_utils.h"
@ -170,30 +169,6 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in
return pos;
}
void BigramDictionary::fillBigramAddressToProbabilityMapAndFilter(const int *prevWord,
const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const {
memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE);
const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot();
int pos = getBigramListPositionForWord(prevWord, prevWordLength,
false /* forceLowerCaseSearch */);
if (0 == pos) {
// If no bigrams for this exact string, search again in lower case.
pos = getBigramListPositionForWord(prevWord, prevWordLength,
true /* forceLowerCaseSearch */);
}
if (0 == pos) return;
uint8_t bigramFlags;
do {
bigramFlags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
const int probability = BinaryFormat::MASK_ATTRIBUTE_PROBABILITY & bigramFlags;
const int bigramPos = BinaryFormat::getAttributeAddressAndForwardPointer(root, bigramFlags,
&pos);
(*map)[bigramPos] = probability;
setInFilter(filter, bigramPos);
} while (BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags);
}
bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) const {
// Checks whether this word starts with same character or neighboring characters of
// what user typed.

6
native/jni/src/suggest/core/dictionary/bigram_dictionary.h
View File

@ -17,9 +17,6 @@
#ifndef LATINIME_BIGRAM_DICTIONARY_H
#define LATINIME_BIGRAM_DICTIONARY_H
#include <map>
#include <stdint.h>
#include "defines.h"
namespace latinime {
@ -32,10 +29,9 @@ class BigramDictionary {
int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords,
int *frequencies, int *outputTypes) const;
void fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength,
std::map<int, int> *map, uint8_t *filter) const;
bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const;
~BigramDictionary();
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary);

1
native/jni/src/suggest/core/dictionary/binary_format.h
View File

@ -19,7 +19,6 @@
#include <stdint.h>
#include "suggest/core/dictionary/bloom_filter.h"
#include "suggest/core/dictionary/probability_utils.h"
#include "utils/char_utils.h"
#include "utils/hash_map_compat.h"

19
native/jni/src/suggest/core/dictionary/probability_utils.h
View File

@ -17,7 +17,6 @@
#ifndef LATINIME_PROBABILITY_UTILS_H
#define LATINIME_PROBABILITY_UTILS_H
#include <map>
#include <stdint.h>
#include "defines.h"
@ -49,24 +48,6 @@ class ProbabilityUtils {
+ static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
}
// This returns a probability in log space.
static AK_FORCE_INLINE int getProbability(const int position,
const std::map<int, int> *const bigramMap,
const uint8_t *bigramFilter, const int unigramProbability) {
if (!bigramMap || !bigramFilter) {
return backoff(unigramProbability);
}
if (!isInFilter(bigramFilter, position)){
return backoff(unigramProbability);
}
const std::map<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position);
if (bigramProbabilityIt != bigramMap->end()) {
const int bigramProbability = bigramProbabilityIt->second;
return computeProbabilityForBigram(unigramProbability, bigramProbability);
}
return backoff(unigramProbability);
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
};

5
native/jni/src/suggest/core/layout/proximity_info.h
View File

@ -24,8 +24,6 @@
namespace latinime {
class Correction;
class ProximityInfo {
public:
ProximityInfo(JNIEnv *env, const jstring localeJStr,
@ -41,7 +39,6 @@ class ProximityInfo {
float getNormalizedSquaredDistanceFromCenterFloatG(
const int keyId, const int x, const int y,
const float verticalScale) const;
bool sameAsTyped(const unsigned short *word, int length) const;
int getCodePointOf(const int keyIndex) const;
bool hasSweetSpotData(const int keyIndex) const {
// When there are no calibration data for a key,
@ -95,8 +92,6 @@ class ProximityInfo {
DISALLOW_IMPLICIT_CONSTRUCTORS(ProximityInfo);
void initializeG();
float calculateNormalizedSquaredDistance(const int keyIndex, const int inputIndex) const;
bool hasInputCoordinates() const;
const int GRID_WIDTH;
const int GRID_HEIGHT;

25
native/jni/src/suggest/core/layout/proximity_info_state.cpp
View File

@ -156,11 +156,6 @@ void ProximityInfoState::initInputParams(const int pointerId, const float maxPoi
if (!isGeometric && pointerId == 0) {
ProximityInfoStateUtils::initPrimaryInputWord(
inputSize, mInputProximities, mPrimaryInputWord);
if (mTouchPositionCorrectionEnabled) {
ProximityInfoStateUtils::initNormalizedSquaredDistances(
mProximityInfo, inputSize, xCoordinates, yCoordinates, mInputProximities,
&mSampledInputXs, &mSampledInputYs, mNormalizedSquaredDistances);
}
}
if (DEBUG_GEO_FULL) {
AKLOGI("ProximityState init finished: %d points out of %d", mSampledInputSize, inputSize);
@ -279,26 +274,6 @@ float ProximityInfoState::getDirection(const int index0, const int index1) const
&mSampledInputXs, &mSampledInputYs, index0, index1);
}
float ProximityInfoState::getLineToKeyDistance(
const int from, const int to, const int keyId, const bool extend) const {
if (from < 0 || from > mSampledInputSize - 1) {
return 0.0f;
}
if (to < 0 || to > mSampledInputSize - 1) {
return 0.0f;
}
const int x0 = mSampledInputXs[from];
const int y0 = mSampledInputYs[from];
const int x1 = mSampledInputXs[to];
const int y1 = mSampledInputYs[to];
const int keyX = mProximityInfo->getKeyCenterXOfKeyIdG(keyId);
const int keyY = mProximityInfo->getKeyCenterYOfKeyIdG(keyId);
return ProximityInfoUtils::pointToLineSegSquaredDistanceFloat(
keyX, keyY, x0, y0, x1, y1, extend);
}
float ProximityInfoState::getMostProbableString(int *const codePointBuf) const {
memcpy(codePointBuf, mMostProbableString, sizeof(mMostProbableString));
return mMostProbableStringProbability;

35
native/jni/src/suggest/core/layout/proximity_info_state.h
View File

@ -53,7 +53,6 @@ class ProximityInfoState {
mSampledSearchKeyVectors(), mTouchPositionCorrectionEnabled(false),
mSampledInputSize(0), mMostProbableStringProbability(0.0f) {
memset(mInputProximities, 0, sizeof(mInputProximities));
memset(mNormalizedSquaredDistances, 0, sizeof(mNormalizedSquaredDistances));
memset(mPrimaryInputWord, 0, sizeof(mPrimaryInputWord));
memset(mMostProbableString, 0, sizeof(mMostProbableString));
}
@ -91,6 +90,19 @@ class ProximityInfoState {
return false;
}
// TODO: Promote insertion letter correction if that letter is a proximity of the previous
// letter like follows:
// // Demotion for a word with excessive character
// if (excessiveCount > 0) {
// multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq);
// if (!lastCharExceeded
// && !proximityInfoState->existsAdjacentProximityChars(excessivePos)) {
// // If an excessive character is not adjacent to the left char or the right char,
// // we will demote this word.
// multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE,
// &finalFreq);
// }
// }
inline bool existsAdjacentProximityChars(const int index) const {
if (index < 0 || index >= mSampledInputSize) return false;
const int currentCodePoint = getPrimaryCodePointAt(index);
@ -106,12 +118,6 @@ class ProximityInfoState {
return false;
}
inline int getNormalizedSquaredDistance(
const int inputIndex, const int proximityIndex) const {
return mNormalizedSquaredDistances[
inputIndex * MAX_PROXIMITY_CHARS_SIZE + proximityIndex];
}
inline const int *getPrimaryInputWord() const {
return mPrimaryInputWord;
}
@ -190,24 +196,10 @@ class ProximityInfoState {
float getProbability(const int index, const int charCode) const;
float getLineToKeyDistance(
const int from, const int to, const int keyId, const bool extend) const;
bool isKeyInSerchKeysAfterIndex(const int index, const int keyId) const;
private:
DISALLOW_COPY_AND_ASSIGN(ProximityInfoState);
/////////////////////////////////////////
// Defined in proximity_info_state.cpp //
/////////////////////////////////////////
float calculateNormalizedSquaredDistance(const int keyIndex, const int inputIndex) const;
float calculateSquaredDistanceFromSweetSpotCenter(
const int keyIndex, const int inputIndex) const;
/////////////////////////////////////////
// Defined here //
/////////////////////////////////////////
inline const int *getProximityCodePointsAt(const int index) const {
return ProximityInfoStateUtils::getProximityCodePointsAt(mInputProximities, index);
@ -249,7 +241,6 @@ class ProximityInfoState {
std::vector<std::vector<int> > mSampledSearchKeyVectors;
bool mTouchPositionCorrectionEnabled;
int mInputProximities[MAX_PROXIMITY_CHARS_SIZE * MAX_WORD_LENGTH];
int mNormalizedSquaredDistances[MAX_PROXIMITY_CHARS_SIZE * MAX_WORD_LENGTH];
int mSampledInputSize;
int mPrimaryInputWord[MAX_WORD_LENGTH];
float mMostProbableStringProbability;

42
native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp
View File

@ -181,48 +181,6 @@ namespace latinime {
return squaredDistance / squaredRadius;
}
/* static */ void ProximityInfoStateUtils::initNormalizedSquaredDistances(
const ProximityInfo *const proximityInfo, const int inputSize, const int *inputXCoordinates,
const int *inputYCoordinates, const int *const inputProximities,
const std::vector<int> *const sampledInputXs, const std::vector<int> *const sampledInputYs,
int *normalizedSquaredDistances) {
memset(normalizedSquaredDistances, NOT_A_DISTANCE,
sizeof(normalizedSquaredDistances[0]) * MAX_PROXIMITY_CHARS_SIZE * MAX_WORD_LENGTH);
const bool hasInputCoordinates = sampledInputXs->size() > 0 && sampledInputYs->size() > 0;
for (int i = 0; i < inputSize; ++i) {
const int *proximityCodePoints = getProximityCodePointsAt(inputProximities, i);
const int primaryKey = proximityCodePoints[0];
const int x = inputXCoordinates[i];
const int y = inputYCoordinates[i];
if (DEBUG_PROXIMITY_CHARS) {
int a = x + y + primaryKey;
a += 0;
AKLOGI("--- Primary = %c, x = %d, y = %d", primaryKey, x, y);
}
for (int j = 0; j < MAX_PROXIMITY_CHARS_SIZE && proximityCodePoints[j] > 0; ++j) {
const int currentCodePoint = proximityCodePoints[j];
const float squaredDistance =
hasInputCoordinates ? calculateNormalizedSquaredDistance(
proximityInfo, sampledInputXs, sampledInputYs,
proximityInfo->getKeyIndexOf(currentCodePoint), i) :
ProximityInfoParams::NOT_A_DISTANCE_FLOAT;
if (squaredDistance >= 0.0f) {
normalizedSquaredDistances[i * MAX_PROXIMITY_CHARS_SIZE + j] =
static_cast<int>(squaredDistance
* ProximityInfoParams::NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR);
} else {
normalizedSquaredDistances[i * MAX_PROXIMITY_CHARS_SIZE + j] =
(j == 0) ? MATCH_CHAR_WITHOUT_DISTANCE_INFO :
PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO;
}
if (DEBUG_PROXIMITY_CHARS) {
AKLOGI("--- Proximity (%d) = %c", j, currentCodePoint);
}
}
}
}
/* static */ void ProximityInfoStateUtils::initGeometricDistanceInfos(
const ProximityInfo *const proximityInfo, const int sampledInputSize,
const int lastSavedInputSize, const float verticalSweetSpotScale,

25
native/jni/src/suggest/core/layout/touch_position_correction_utils.h
View File

@ -23,31 +23,6 @@
namespace latinime {
class TouchPositionCorrectionUtils {
public:
// TODO: (OLD) Remove
static float getLengthScalingFactor(const float normalizedSquaredDistance) {
// Promote or demote the score according to the distance from the sweet spot
static const float A = ZERO_DISTANCE_PROMOTION_RATE / 100.0f;
static const float B = 1.0f;
static const float C = 0.5f;
static const float MIN = 0.3f;
static const float R1 = NEUTRAL_SCORE_SQUARED_RADIUS;
static const float R2 = HALF_SCORE_SQUARED_RADIUS;
const float x = normalizedSquaredDistance / static_cast<float>(
ProximityInfoParams::NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR);
const float factor = max((x < R1)
? (A * (R1 - x) + B * x) / R1
: (B * (R2 - x) + C * (x - R1)) / (R2 - R1), MIN);
// factor is a piecewise linear function like:
// A -_ .
// ^-_ .
// B \ .
// \_ .
// C ------------.
// .
// 0 R1 R2 .
return factor;
}
static float getSweetSpotFactor(const bool isTouchPositionCorrectionEnabled,
const float normalizedSquaredDistance) {
// Promote or demote the score according to the distance from the sweet spot

20
native/jni/src/suggest/policyimpl/utils/edit_distance.h
View File

@ -62,6 +62,26 @@ class EditDistance {
return dp[(beforeLength + 1) * (afterLength + 1) - 1];
}
AK_FORCE_INLINE static void dumpEditDistance10ForDebug(const float *const editDistanceTable,
const int editDistanceTableWidth, const int outputLength) {
if (DEBUG_DICT) {
AKLOGI("EditDistanceTable");
for (int i = 0; i <= 10; ++i) {
float c[11];
for (int j = 0; j <= 10; ++j) {
if (j < editDistanceTableWidth + 1 && i < outputLength + 1) {
c[j] = (editDistanceTable + i * (editDistanceTableWidth + 1))[j];
} else {
c[j] = -1.0f;
}
}
AKLOGI("[ %f, %f, %f, %f, %f, %f, %f, %f, %f, %f, %f ]",
c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7], c[8], c[9], c[10]);
(void)c; // To suppress compiler warning
}
}
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(EditDistance);
};

105
native/jni/src/utils/autocorrection_threshold_utils.cpp Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utils/autocorrection_threshold_utils.h"
#include <cmath>
#include "defines.h"
#include "suggest/policyimpl/utils/edit_distance.h"
#include "suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h"
namespace latinime {
const int AutocorrectionThresholdUtils::MAX_INITIAL_SCORE = 255;
const int AutocorrectionThresholdUtils::TYPED_LETTER_MULTIPLIER = 2;
const int AutocorrectionThresholdUtils::FULL_WORD_MULTIPLIER = 2;
/* static */ int AutocorrectionThresholdUtils::editDistance(const int *before,
const int beforeLength, const int *after, const int afterLength) {
const DamerauLevenshteinEditDistancePolicy daemaruLevenshtein(
before, beforeLength, after, afterLength);
return static_cast<int>(EditDistance::getEditDistance(&daemaruLevenshtein));
}
// In dictionary.cpp, getSuggestion() method,
// When USE_SUGGEST_INTERFACE_FOR_TYPING is true:
//
// // TODO: Revise the following logic thoroughly by referring to the logic
// // marked as "Otherwise" below.
// SUGGEST_INTERFACE_OUTPUT_SCALE was multiplied to the original suggestion scores to convert
// them to integers.
// score = (int)((original score) * SUGGEST_INTERFACE_OUTPUT_SCALE)
// Undo the scaling here to recover the original score.
// normalizedScore = ((float)score) / SUGGEST_INTERFACE_OUTPUT_SCALE
//
// Otherwise: suggestion scores are computed using the below formula.
// original score
// := powf(mTypedLetterMultiplier (this is defined 2),
// (the number of matched characters between typed word and suggested word))
// * (individual word's score which defined in the unigram dictionary,
// and this score is defined in range [0, 255].)
// Then, the following processing is applied.
// - If the dictionary word is matched up to the point of the user entry
// (full match up to min(before.length(), after.length())
// => Then multiply by FULL_MATCHED_WORDS_PROMOTION_RATE (this is defined 1.2)
// - If the word is a true full match except for differences in accents or
// capitalization, then treat it as if the score was 255.
// - If before.length() == after.length()
// => multiply by mFullWordMultiplier (this is defined 2))
// So, maximum original score is powf(2, min(before.length(), after.length())) * 255 * 2 * 1.2
// For historical reasons we ignore the 1.2 modifier (because the measure for a good
// autocorrection threshold was done at a time when it didn't exist). This doesn't change
// the result.
// So, we can normalize original score by dividing powf(2, min(b.l(),a.l())) * 255 * 2.
/* static */ float AutocorrectionThresholdUtils::calcNormalizedScore(const int *before,
const int beforeLength, const int *after, const int afterLength, const int score) {
if (0 == beforeLength || 0 == afterLength) {
return 0.0f;
}
const int distance = editDistance(before, beforeLength, after, afterLength);
int spaceCount = 0;
for (int i = 0; i < afterLength; ++i) {
if (after[i] == KEYCODE_SPACE) {
++spaceCount;
}
}
if (spaceCount == afterLength) {
return 0.0f;
}
// add a weight based on edit distance.
// distance <= max(afterLength, beforeLength) == afterLength,
// so, 0 <= distance / afterLength <= 1
const float weight = 1.0f - static_cast<float>(distance) / static_cast<float>(afterLength);
// TODO: Revise the following logic thoroughly by referring to...
if (true /* USE_SUGGEST_INTERFACE_FOR_TYPING */) {
return (static_cast<float>(score) / SUGGEST_INTERFACE_OUTPUT_SCALE) * weight;
}
// ...this logic.
const float maxScore = score >= S_INT_MAX ? static_cast<float>(S_INT_MAX)
: static_cast<float>(MAX_INITIAL_SCORE)
* powf(static_cast<float>(TYPED_LETTER_MULTIPLIER),
static_cast<float>(min(beforeLength, afterLength - spaceCount)))
* static_cast<float>(FULL_WORD_MULTIPLIER);
return (static_cast<float>(score) / maxScore) * weight;
}
} // namespace latinime

39
native/jni/src/utils/autocorrection_threshold_utils.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_AUTOCORRECTION_THRESHOLD_UTILS_H
#define LATINIME_AUTOCORRECTION_THRESHOLD_UTILS_H
#include "defines.h"
namespace latinime {
class AutocorrectionThresholdUtils {
public:
static float calcNormalizedScore(const int *before, const int beforeLength,
const int *after, const int afterLength, const int score);
static int editDistance(const int *before, const int beforeLength, const int *after,
const int afterLength);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(AutocorrectionThresholdUtils);
static const int MAX_INITIAL_SCORE;
static const int TYPED_LETTER_MULTIPLIER;
static const int FULL_WORD_MULTIPLIER;
};
} // namespace latinime
#endif // LATINIME_AUTOCORRECTION_THRESHOLD_UTILS_H