70 lines
2.8 KiB
C
70 lines
2.8 KiB
C
|
/*
|
||
|
* Copyright (C) 2013, The Android Open Source Project
|
||
|
*
|
||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
* you may not use this file except in compliance with the License.
|
||
|
* You may obtain a copy of the License at
|
||
|
*
|
||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||
|
*
|
||
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
* See the License for the specific language governing permissions and
|
||
|
* limitations under the License.
|
||
|
*/
|
||
|
|
||
|
#ifndef LATINIME_PROBABILITY_UTILS_H
|
||
|
#define LATINIME_PROBABILITY_UTILS_H
|
||
|
|
||
|
#include <algorithm>
|
||
|
#include <cmath>
|
||
|
|
||
|
#include "defines.h"
|
||
|
|
||
|
namespace latinime {
|
||
|
|
||
|
// TODO: Quit using bigram probability to indicate the delta.
|
||
|
class ProbabilityUtils {
|
||
|
public:
|
||
|
static AK_FORCE_INLINE int backoff(const int unigramProbability) {
|
||
|
return unigramProbability;
|
||
|
// For some reason, applying the backoff weight gives bad results in tests. To apply the
|
||
|
// backoff weight, we divide the probability by 2, which in our storing format means
|
||
|
// decreasing the score by 8.
|
||
|
// TODO: figure out what's wrong with this.
|
||
|
// return unigramProbability > 8 ?
|
||
|
// unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
|
||
|
}
|
||
|
|
||
|
static AK_FORCE_INLINE int computeProbabilityForBigram(
|
||
|
const int unigramProbability, const int bigramProbability) {
|
||
|
// We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want
|
||
|
// the unigram probability to be the median value of the 17th step from the top. A value of
|
||
|
// 0 for the bigram probability represents the middle of the 16th step from the top,
|
||
|
// while a value of 15 represents the middle of the top step.
|
||
|
// See makedict.BinaryDictEncoder#makeBigramFlags for details.
|
||
|
const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
|
||
|
/ (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
|
||
|
return unigramProbability
|
||
|
+ static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
|
||
|
}
|
||
|
|
||
|
// Encode probability using the same way as we are doing for main dictionaries.
|
||
|
static AK_FORCE_INLINE int encodeRawProbability(const float rawProbability) {
|
||
|
const float probability = static_cast<float>(MAX_PROBABILITY)
|
||
|
+ log2f(rawProbability) * PROBABILITY_ENCODING_SCALER;
|
||
|
if (probability < 0.0f) {
|
||
|
return 0;
|
||
|
}
|
||
|
return std::min(static_cast<int>(probability + 0.5f), MAX_PROBABILITY);
|
||
|
}
|
||
|
|
||
|
private:
|
||
|
DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
|
||
|
|
||
|
static const float PROBABILITY_ENCODING_SCALER;
|
||
|
};
|
||
|
}
|
||
|
#endif /* LATINIME_PROBABILITY_UTILS_H */
|