nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Introduce WordAttributes to get word probability and flags."

main
Keisuke Kuroyanagi 2014-09-10 12:03:43 +00:00 committed by Android (Google) Code Review
parent 6cc1cb6372 2111e3abc9
commit 6443d799ee
10 changed files with 116 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
View File

@ -72,10 +72,10 @@ namespace latinime {
if (dicNode->hasMultipleWords() && !dicNode->isValidMultipleWordSuggestion()) {
return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
const int probability = dictionaryStructurePolicy->getProbabilityOfWordInContext(
const WordAttributes wordAttributes = dictionaryStructurePolicy->getWordAttributesInContext(
dicNode->getPrevWordIds(), dicNode->getWordId(), multiBigramMap);
// TODO: This equation to calculate the improbability looks unreasonable. Investigate this.
const float cost = static_cast<float>(MAX_PROBABILITY - probability)
const float cost = static_cast<float>(MAX_PROBABILITY - wordAttributes.getProbability())
/ static_cast<float>(MAX_PROBABILITY);
return cost;
}

7
native/jni/src/suggest/core/dictionary/dictionary.cpp
View File

@ -84,9 +84,10 @@ void Dictionary::NgramListenerForPrediction::onVisitEntry(const int ngramProbabi
if (codePointCount <= 0) {
return;
}
const int probability = mDictStructurePolicy->getProbabilityOfWordInContext(mPrevWordIds.data(),
targetWordId, nullptr /* multiBigramMap */);
mSuggestionResults->addPrediction(targetWordCodePoints, codePointCount, probability);
const WordAttributes wordAttributes = mDictStructurePolicy->getWordAttributesInContext(
mPrevWordIds.data(), targetWordId, nullptr /* multiBigramMap */);
mSuggestionResults->addPrediction(targetWordCodePoints, codePointCount,
wordAttributes.getProbability());
}
void Dictionary::getPredictions(const PrevWordsInfo *const prevWordsInfo,

60
native/jni/src/suggest/core/dictionary/word_attributes.h Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (C) 2014, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_WORD_ATTRIBUTES_H
#define LATINIME_WORD_ATTRIBUTES_H
#include "defines.h"
class WordAttributes {
public:
// Invalid word attributes.
WordAttributes()
: mProbability(NOT_A_PROBABILITY), mIsBlacklisted(false), mIsNotAWord(false),
mIsPossiblyOffensive(false) {}
WordAttributes(const int probability, const bool isBlacklisted, const bool isNotAWord,
const bool isPossiblyOffensive)
: mProbability(probability), mIsBlacklisted(isBlacklisted), mIsNotAWord(isNotAWord),
mIsPossiblyOffensive(isPossiblyOffensive) {}
int getProbability() const {
return mProbability;
}
bool isBlacklisted() const {
return mIsBlacklisted;
}
bool isNotAWord() const {
return mIsNotAWord;
}
bool isPossiblyOffensive() const {
return mIsPossiblyOffensive;
}
private:
DISALLOW_ASSIGNMENT_OPERATOR(WordAttributes);
int mProbability;
bool mIsBlacklisted;
bool mIsNotAWord;
bool mIsPossiblyOffensive;
};
// namespace
#endif /* LATINIME_WORD_ATTRIBUTES_H */

5
native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h
View File

@ -22,6 +22,7 @@
#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_shortcut_iterator.h"
#include "suggest/core/dictionary/property/word_property.h"
#include "suggest/core/dictionary/word_attributes.h"
#include "utils/int_array_view.h"
namespace latinime {
@ -57,8 +58,8 @@ class DictionaryStructureWithBufferPolicy {
virtual int getWordId(const CodePointArrayView wordCodePoints,
const bool forceLowerCaseSearch) const = 0;
virtual int getProbabilityOfWordInContext(const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const = 0;
virtual const WordAttributes getWordAttributesInContext(const int *const prevWordIds,
const int wordId, MultiBigramMap *const multiBigramMap) const = 0;
// TODO: Remove
virtual int getProbability(const int unigramProbability, const int bigramProbability) const = 0;

23
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp
View File

@ -118,24 +118,33 @@ int Ver4PatriciaTriePolicy::getWordId(const CodePointArrayView wordCodePoints,
return getWordIdFromTerminalPtNodePos(ptNodePos);
}
int Ver4PatriciaTriePolicy::getProbabilityOfWordInContext(const int *const prevWordIds,
const int wordId, MultiBigramMap *const multiBigramMap) const {
const WordAttributes Ver4PatriciaTriePolicy::getWordAttributesInContext(
const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const {
if (wordId == NOT_A_WORD_ID) {
return NOT_A_PROBABILITY;
return WordAttributes();
}
const int ptNodePos = getTerminalPtNodePosFromWordId(wordId);
const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
if (multiBigramMap) {
return multiBigramMap->getBigramProbability(this /* structurePolicy */, prevWordIds,
wordId, ptNodeParams.getProbability());
const int probability = multiBigramMap->getBigramProbability(this /* structurePolicy */,
prevWordIds, wordId, ptNodeParams.getProbability());
return getWordAttributes(probability, ptNodeParams);
}
if (prevWordIds) {
const int probability = getProbabilityOfWord(prevWordIds, wordId);
if (probability != NOT_A_PROBABILITY) {
return probability;
return getWordAttributes(probability, ptNodeParams);
}
}
return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY);
return getWordAttributes(getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY),
ptNodeParams);
}
const WordAttributes Ver4PatriciaTriePolicy::getWordAttributes(const int probability,
const PtNodeParams &ptNodeParams) const {
return WordAttributes(probability, ptNodeParams.isBlacklisted(), ptNodeParams.isNotAWord(),
ptNodeParams.getProbability() == 0);
}
int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,

4
native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h
View File

@ -91,7 +91,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getWordId(const CodePointArrayView wordCodePoints, const bool forceLowerCaseSearch) const;
int getProbabilityOfWordInContext(const int *const prevWordIds, const int wordId,
const WordAttributes getWordAttributesInContext(const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;
@ -166,6 +166,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getShortcutPositionOfPtNode(const int ptNodePos) const;
int getWordIdFromTerminalPtNodePos(const int ptNodePos) const;
int getTerminalPtNodePosFromWordId(const int wordId) const;
const WordAttributes getWordAttributes(const int probability,
const PtNodeParams &ptNodeParams) const;
};
} // namespace v402
} // namespace backward

20
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp
View File

@ -282,25 +282,33 @@ int PatriciaTriePolicy::getWordId(const CodePointArrayView wordCodePoints,
return getWordIdFromTerminalPtNodePos(ptNodePos);
}
int PatriciaTriePolicy::getProbabilityOfWordInContext(const int *const prevWordIds,
const WordAttributes PatriciaTriePolicy::getWordAttributesInContext(const int *const prevWordIds,
const int wordId, MultiBigramMap *const multiBigramMap) const {
if (wordId == NOT_A_WORD_ID) {
return NOT_A_PROBABILITY;
return WordAttributes();
}
const int ptNodePos = getTerminalPtNodePosFromWordId(wordId);
const PtNodeParams ptNodeParams =
mPtNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
if (multiBigramMap) {
return multiBigramMap->getBigramProbability(this /* structurePolicy */, prevWordIds,
wordId, ptNodeParams.getProbability());
const int probability = multiBigramMap->getBigramProbability(this /* structurePolicy */,
prevWordIds, wordId, ptNodeParams.getProbability());
return getWordAttributes(probability, ptNodeParams);
}
if (prevWordIds) {
const int bigramProbability = getProbabilityOfWord(prevWordIds, wordId);
if (bigramProbability != NOT_A_PROBABILITY) {
return bigramProbability;
return getWordAttributes(bigramProbability, ptNodeParams);
}
}
return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY);
return getWordAttributes(getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY),
ptNodeParams);
}
const WordAttributes PatriciaTriePolicy::getWordAttributes(const int probability,
const PtNodeParams &ptNodeParams) const {
return WordAttributes(probability, ptNodeParams.isBlacklisted(), ptNodeParams.isNotAWord(),
ptNodeParams.getProbability() == 0);
}
int PatriciaTriePolicy::getProbability(const int unigramProbability,

4
native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h
View File

@ -66,7 +66,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getWordId(const CodePointArrayView wordCodePoints, const bool forceLowerCaseSearch) const;
int getProbabilityOfWordInContext(const int *const prevWordIds, const int wordId,
const WordAttributes getWordAttributesInContext(const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;
@ -163,6 +163,8 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
DicNodeVector *const childDicNodes) const;
int getWordIdFromTerminalPtNodePos(const int ptNodePos) const;
int getTerminalPtNodePosFromWordId(const int wordId) const;
const WordAttributes getWordAttributes(const int probability,
const PtNodeParams &ptNodeParams) const;
};
} // namespace latinime
#endif // LATINIME_PATRICIA_TRIE_POLICY_H

15
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View File

@ -113,14 +113,19 @@ int Ver4PatriciaTriePolicy::getWordId(const CodePointArrayView wordCodePoints,
return ptNodeParams.getTerminalId();
}
int Ver4PatriciaTriePolicy::getProbabilityOfWordInContext(const int *const prevWordIds,
const int wordId, MultiBigramMap *const multiBigramMap) const {
const WordAttributes Ver4PatriciaTriePolicy::getWordAttributesInContext(
const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const {
if (wordId == NOT_A_WORD_ID) {
return NOT_A_PROBABILITY;
return WordAttributes();
}
const int ptNodePos =
mBuffers->getTerminalPositionLookupTable()->getTerminalPtNodePosition(wordId);
const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
// TODO: Support n-gram.
return mBuffers->getLanguageModelDictContent()->getWordProbability(
WordIdArrayView::singleElementView(prevWordIds), wordId);
return WordAttributes(mBuffers->getLanguageModelDictContent()->getWordProbability(
WordIdArrayView::singleElementView(prevWordIds), wordId), ptNodeParams.isBlacklisted(),
ptNodeParams.isNotAWord(), ptNodeParams.getProbability() == 0);
}
int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View File

@ -68,7 +68,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getWordId(const CodePointArrayView wordCodePoints, const bool forceLowerCaseSearch) const;
int getProbabilityOfWordInContext(const int *const prevWordIds, const int wordId,
const WordAttributes getWordAttributesInContext(const int *const prevWordIds, const int wordId,
MultiBigramMap *const multiBigramMap) const;
int getProbability(const int unigramProbability, const int bigramProbability) const;