nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Quit use bigram probability diff for ver4 dict.

Browse source 
Change-Id: I2cfcfbcf351877d1dff466a24974dbb05908f14e
This commit is contained in:
Keisuke Kuroyanagi 2014-05-15 16:02:58 +09:00
parent 04536f14b0
commit 64341927d2
6 changed files with 129 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -214,8 +214,6 @@ public final class BinaryDictionary extends Dictionary {
private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
private static native int addMultipleDictionaryEntriesNative(long dict,
LanguageModelParam[] languageModelParams, int startIndex);
private static native int calculateProbabilityNative(long dict, int unigramProbability,
int bigramProbability);
private static native String getPropertyNative(long dict, String query);
private static native boolean isCorruptedNative(long dict);
private static native boolean migrateNative(long dict, String dictFilePath,
@ -551,12 +549,6 @@ public final class BinaryDictionary extends Dictionary {
return true;
}
@UsedForTesting
public int calculateProbability(final int unigramProbability, final int bigramProbability) {
if (!isValidDictionary()) return NOT_A_PROBABILITY;
return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
}
@UsedForTesting
public String getPropertyForTest(final String query) {
if (!isValidDictionary()) return "";

15
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -467,16 +467,6 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
return languageModelParamCount;
}
static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jclass clazz,
jlong dict, jint unigramProbability, jint bigramProbability) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) {
return NOT_A_PROBABILITY;
}
return dictionary->getDictionaryStructurePolicy()->getProbability(unigramProbability,
bigramProbability);
}
static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
jstring query) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
@ -669,11 +659,6 @@ static const JNINativeMethod sMethods[] = {
"(J[Lcom/android/inputmethod/latin/utils/LanguageModelParam;I)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_addMultipleDictionaryEntries)
},
{
const_cast<char *>("calculateProbabilityNative"),
const_cast<char *>("(JII)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
},
{
const_cast<char *>("getPropertyNative"),
const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),

22
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp
View file

@ -38,8 +38,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
int level = 0;
int count = 0;
if (mHasHistoricalInfo) {
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
timestamp = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
level = bigramListBuffer->readUintAndAdvancePosition(
@ -47,7 +45,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
count = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
} else {
probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
probability = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
}
const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@ -65,21 +64,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
bool BigramDictContent::writeBigramEntryAndAdvancePosition(
const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
const int bigramFlags = createAndGetBigramFlags(
mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
bigramEntryToWrite->hasNext());
const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
return false;
}
if (mHasHistoricalInfo) {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@ -99,6 +90,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
historicalInfo->getCount());
return false;
}
} else {
if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
bigramEntryToWrite->getProbability());
return false;
}
}
const int targetTerminalIdToWrite =
(bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?

5
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h
View file

@ -95,9 +95,8 @@ class BigramDictContent : public SparseTableDictContent {
private:
DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
int createAndGetBigramFlags(const int probability, const bool hasNext) const {
return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
| (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
int createAndGetBigramFlags(const bool hasNext) const {
return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
}
bool runGCBigramList(const int bigramListPos,

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
View file

@ -115,9 +115,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
} else if (bigramProbability == NOT_A_PROBABILITY) {
return ProbabilityUtils::backoff(unigramProbability);
} else {
// bigramProbability is a bigram probability delta.
return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
bigramProbability);
return bigramProbability;
}
}
}
@ -398,7 +396,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
const int probability = bigramEntry.hasHistoricalInfo() ?
ForgettingCurveUtils::decodeProbability(
bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
getProbability(word1Probability, bigramEntry.getProbability());
bigramEntry.getProbability();
bigrams.emplace_back(&word1, probability,
historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
historicalInfo->getCount());

195
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View file

@ -46,6 +46,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private static final int[] DICT_FORMAT_VERSIONS =
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
private static boolean canCheckBigramProbability(final int formatVersion) {
return formatVersion >= FormatSpec.VERSION4_DEV;
}
private File createEmptyDictionaryAndGetFile(final String dictId,
final int formatVersion) throws IOException {
if (formatVersion == FormatSpec.VERSION4
@ -298,8 +302,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
final int bigramProbability = 10;
final int updatedBigramProbability = 15;
final int bigramProbability = 150;
final int updatedBigramProbability = 200;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@ -308,25 +312,26 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
final int probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
}
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability,
updatedBigramProbability);
assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability,
binaryDictionary.getBigramProbability("aaa", "abb"));
}
assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("bcc", "aaa"));
assertEquals(Dictionary.NOT_A_PROBABILITY,
@ -341,11 +346,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "fgh", unigramProbability);
addUnigramWord(binaryDictionary, "abc", unigramProbability);
addUnigramWord(binaryDictionary, "f", unigramProbability);
assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
binaryDictionary.getBigramProbability("abcde", "fghij"));
}
assertEquals(Dictionary.NOT_A_PROBABILITY,
binaryDictionary.getBigramProbability("abcde", "fgh"));
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(updatedBigramProbability,
binaryDictionary.getBigramProbability("abcde", "fghij"));
}
dictFile.delete();
}
@ -396,18 +408,21 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
bigramProbabilities.put(bigram, bigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
for (final Pair<String, String> bigram : bigramWords) {
final int unigramProbability = unigramProbabilities.get(bigram.second);
final int bigramProbability = bigramProbabilities.get(bigram);
final int probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
assertEquals(probability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
}
dictFile.delete();
@ -430,7 +445,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
final int bigramProbability = 10;
final int bigramProbability = 150;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@ -439,23 +454,23 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability);
addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability);
assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bcc"));
assertTrue(binaryDictionary.isValidBigram("abb", "aaa"));
assertTrue(binaryDictionary.isValidBigram("abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb");
assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb"));
assertFalse(binaryDictionary.isValidBigram("aaa", "abb"));
addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability);
assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "abb"));
binaryDictionary.removeBigramWords("aaa", "bcc");
assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "bcc"));
binaryDictionary.removeBigramWords("abb", "aaa");
assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa"));
assertFalse(binaryDictionary.isValidBigram("abb", "aaa"));
binaryDictionary.removeBigramWords("abb", "bcc");
assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc"));
assertFalse(binaryDictionary.isValidBigram("abb", "bcc"));
binaryDictionary.removeBigramWords("aaa", "abb");
// Test remove non-existing bigram operation.
@ -537,7 +552,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
final int bigramProbability = 10;
final int bigramProbability = 150;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "abb", unigramProbability);
addUnigramWord(binaryDictionary, "bcc", unigramProbability);
@ -551,18 +566,18 @@ public class BinaryDictionaryTests extends AndroidTestCase {
binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc"));
assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa"));
assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc"));
assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa"));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "abb"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bcc"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "aaa"));
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("abb", "bcc"));
}
assertFalse(binaryDictionary.isValidBigram("bcc", "aaa"));
assertFalse(binaryDictionary.isValidBigram("bcc", "bbc"));
assertFalse(binaryDictionary.isValidBigram("aaa", "aaa"));
binaryDictionary.flushWithGC();
binaryDictionary.close();
@ -617,7 +632,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
bigramProbabilities.put(bigram, bigramProbability);
addBigramWords(binaryDictionary, word0, word1, bigramProbability);
}
@ -628,13 +645,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
for (final Pair<String, String> bigram : bigramWords) {
final int unigramProbability = unigramProbabilities.get(bigram.second);
final int bigramProbability = bigramProbabilities.get(bigram);
final int probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
assertEquals(probability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
}
dictFile.delete();
@ -709,7 +728,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
if (TextUtils.equals(word0, word1)) {
continue;
}
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
bigramWords.add(bigram);
bigramProbabilities.put(bigram, bigramProbability);
@ -734,17 +755,20 @@ public class BinaryDictionaryTests extends AndroidTestCase {
// Test whether the all bigram operations are collectlly handled.
for (int i = 0; i < bigramWords.size(); i++) {
final Pair<String, String> bigram = bigramWords.get(i);
final int unigramProbability = unigramProbabilities.get(bigram.second);
final int probability;
if (bigramProbabilities.containsKey(bigram)) {
final int bigramProbability = bigramProbabilities.get(bigram);
probability = binaryDictionary.calculateProbability(unigramProbability,
bigramProbability);
probability = bigramProbability;
} else {
probability = Dictionary.NOT_A_PROBABILITY;
}
assertEquals(probability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(probability,
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
binaryDictionary.flushWithGC();
binaryDictionary.close();
@ -894,7 +918,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int i = 0; i < languageModelParams.length; i++) {
final String word = CodePointUtils.generateWord(random, codePointSet);
final int probability = random.nextInt(0xFF);
final int bigramProbability = random.nextInt(0xF);
final int bigramProbability = probability + random.nextInt(0xFF - probability);
unigramProbabilities.put(word, probability);
if (prevWord == null) {
languageModelParams[i] = new LanguageModelParam(word, probability,
@ -920,11 +944,13 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (Map.Entry<Pair<String, String>, Integer> entry : bigramProbabilities.entrySet()) {
final String word0 = entry.getKey().first;
final String word1 = entry.getKey().second;
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability = entry.getValue();
final int probability = binaryDictionary.calculateProbability(
unigramProbability, bigramProbability);
assertEquals(probability, binaryDictionary.getBigramProbability(word0, word1));
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
binaryDictionary.isValidBigram(word0, word1));
if (canCheckBigramProbability(formatVersion)) {
assertEquals(bigramProbability,
binaryDictionary.getBigramProbability(word0, word1));
}
}
}
@ -994,7 +1020,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = wordProbabilities.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@ -1019,12 +1047,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int bigramProbabilityDelta = bigramProbabilities.get(
new Pair<String, String>(word0, word1));
final int unigramProbability = wordProbabilities.get(word1);
final int bigramProbablity = binaryDictionary.calculateProbability(
unigramProbability, bigramProbabilityDelta);
assertEquals(wordProperty.mBigrams.get(j).getProbability(), bigramProbablity);
if (canCheckBigramProbability(formatVersion)) {
final int bigramProbability = bigramProbabilities.get(
new Pair<String, String>(word0, word1));
assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability());
}
}
}
}
@ -1082,7 +1109,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
final int bigramProbability =
unigramProbability + random.nextInt(0xFF - unigramProbability);
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@ -1113,12 +1142,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int unigramProbability = wordProbabilitiesToCheckLater.get(word1);
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
final int bigramProbabilityDelta = bigramProbabilitiesToCheckLater.get(bigram);
final int bigramProbablity = binaryDictionary.calculateProbability(
unigramProbability, bigramProbabilityDelta);
assertEquals(wordProperty.mBigrams.get(j).getProbability(), bigramProbablity);
if (canCheckBigramProbability(formatVersion)) {
final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram);
assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability());
}
bigramSet.remove(bigram);
}
token = result.mNextToken;
@ -1286,7 +1314,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int unigramProbability = 100;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "bbb", unigramProbability);
final int bigramProbability = 10;
final int bigramProbability = 150;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
final int shortcutProbability = 10;
binaryDictionary.addUnigramWord("ccc", unigramProbability, "xxx", shortcutProbability,
@ -1303,7 +1331,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
// TODO: Add tests for bigram frequency when the implementation gets ready.
if (canCheckBigramProbability(toFormatVersion)) {
assertEquals(bigramProbability, binaryDictionary.getBigramProbability("aaa", "bbb"));
}
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc");
assertEquals(1, wordProperty.mShortcutTargets.size());
@ -1362,7 +1392,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
final String word0 = words.get(word0Index);
final String word1 = words.get(word1Index);
final int bigramProbability = random.nextInt(0xF);
final int unigramProbability = unigramProbabilities.get(word1);
final int bigramProbability =
random.nextInt(0xFF - unigramProbability) + unigramProbability;
binaryDictionary.addBigramWords(word0, word1, bigramProbability,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
@ -1381,7 +1413,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
for (final Pair<String, String> bigram : bigrams) {
// TODO: Add tests for bigram frequency when the implementation gets ready.
if (canCheckBigramProbability(toFormatVersion)) {
assertEquals((int)bigramProbabilities.get(bigram),
binaryDictionary.getBigramProbability(bigram.first, bigram.second));
}
assertTrue(binaryDictionary.isValidBigram(bigram.first, bigram.second));
}
assertEquals(bigramProbabilities.size(), Integer.parseInt(