am b3bae2e8: Merge "Update v4 format version from 402 to 403."

* commit 'b3bae2e89bdd6046ff6cc1fd0fe4be889e66ec17':
  Update v4 format version from 402 to 403.
main
Keisuke Kuroyanagi 2014-10-31 14:22:32 +00:00 committed by Android Git Automerger
commit 87af779f84
11 changed files with 73 additions and 112 deletions

View File

@ -120,7 +120,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
private static boolean needsToMigrateDictionary(final int formatVersion) { private static boolean needsToMigrateDictionary(final int formatVersion) {
// When we bump up the dictionary format version, the old version should be added to here // When we bump up the dictionary format version, the old version should be added to here
// for supporting migration. Note that native code has to support reading such formats. // for supporting migration. Note that native code has to support reading such formats.
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING; return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|| formatVersion == FormatSpec.VERSION402;
} }
public boolean isValidDictionaryLocked() { public boolean isValidDictionaryLocked() {

View File

@ -175,9 +175,10 @@ public final class FormatSpec {
public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201; public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201;
// Dictionary version used for testing. // Dictionary version used for testing.
public static final int VERSION4_ONLY_FOR_TESTING = 399; public static final int VERSION4_ONLY_FOR_TESTING = 399;
public static final int VERSION401 = 401; public static final int VERSION402 = 402;
public static final int VERSION4 = 402; public static final int VERSION403 = 403;
public static final int VERSION4_DEV = 403; public static final int VERSION4 = VERSION403;
public static final int VERSION4_DEV = VERSION403;
static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202; static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202; static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4; static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4;

View File

@ -141,10 +141,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return FormatUtils::VERSION_202; return FormatUtils::VERSION_202;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
return FormatUtils::VERSION_4_ONLY_FOR_TESTING; return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
case FormatUtils::VERSION_4: case FormatUtils::VERSION_402:
return FormatUtils::VERSION_4; return FormatUtils::VERSION_402;
case FormatUtils::VERSION_4_DEV: case FormatUtils::VERSION_403:
return FormatUtils::VERSION_4_DEV; return FormatUtils::VERSION_403;
default: default:
return FormatUtils::UNKNOWN_VERSION; return FormatUtils::UNKNOWN_VERSION;
} }
@ -247,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
} }
bool supportsBeginningOfSentence() const { bool supportsBeginningOfSentence() const {
return mDictFormatVersion >= FormatUtils::VERSION_4; return mDictFormatVersion >= FormatUtils::VERSION_402;
} }
const int *getCodePointTable() const { const int *getCodePointTable() const {

View File

@ -115,8 +115,8 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
// None of the static dictionaries (v2x) support writing // None of the static dictionaries (v2x) support writing
return false; return false;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: case FormatUtils::VERSION_402:
case FormatUtils::VERSION_4_DEV: case FormatUtils::VERSION_403:
return buffer->writeUintAndAdvancePosition(version /* data */, return buffer->writeUintAndAdvancePosition(version /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos); HEADER_DICTIONARY_VERSION_SIZE, writingPos);
default: default:

View File

@ -58,7 +58,7 @@ namespace latinime {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion); FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
switch (dictFormatVersion) { switch (dictFormatVersion) {
case FormatUtils::VERSION_4: { case FormatUtils::VERSION_402: {
return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants, return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers, backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr, backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
@ -66,7 +66,7 @@ namespace latinime {
dictFormatVersion, locale, attributeMap); dictFormatVersion, locale, attributeMap);
} }
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: { case FormatUtils::VERSION_403: {
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers, return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>( Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
dictFormatVersion, locale, attributeMap); dictFormatVersion, locale, attributeMap);
@ -118,7 +118,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
case FormatUtils::VERSION_202: case FormatUtils::VERSION_202:
AKLOGE("Given path is a directory but the format is version 2xx. path: %s", path); AKLOGE("Given path is a directory but the format is version 2xx. path: %s", path);
break; break;
case FormatUtils::VERSION_4: { case FormatUtils::VERSION_402: {
return newPolicyForV4Dict<backward::v402::Ver4DictConstants, return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers, backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr, backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
@ -126,7 +126,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
headerFilePath, formatVersion, std::move(mmappedBuffer)); headerFilePath, formatVersion, std::move(mmappedBuffer));
} }
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: { case FormatUtils::VERSION_403: {
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers, return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>( Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
headerFilePath, formatVersion, std::move(mmappedBuffer)); headerFilePath, formatVersion, std::move(mmappedBuffer));
@ -184,8 +184,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer))); new PatriciaTriePolicy(std::move(mmappedBuffer)));
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: case FormatUtils::VERSION_402:
case FormatUtils::VERSION_4_DEV: case FormatUtils::VERSION_403:
AKLOGE("Given path is a file but the format is version 4. path: %s", path); AKLOGE("Given path is a file but the format is version 4. path: %s", path);
break; break;
default: default:

View File

@ -44,13 +44,13 @@ const int DictFileWritingUtils::SIZE_OF_BUFFER_SIZE_FIELD = 4;
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion); const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
switch (formatVersion) { switch (formatVersion) {
case FormatUtils::VERSION_4: case FormatUtils::VERSION_402:
return createEmptyV4DictFile<backward::v402::Ver4DictConstants, return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
backward::v402::Ver4DictBuffers, backward::v402::Ver4DictBuffers,
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>( backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion); filePath, localeAsCodePointVector, attributeMap, formatVersion);
case FormatUtils::VERSION_4_ONLY_FOR_TESTING: case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4_DEV: case FormatUtils::VERSION_403:
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers, return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,
Ver4DictBuffers::Ver4DictBuffersPtr>( Ver4DictBuffers::Ver4DictBuffersPtr>(
filePath, localeAsCodePointVector, attributeMap, formatVersion); filePath, localeAsCodePointVector, attributeMap, formatVersion);

View File

@ -35,10 +35,10 @@ const size_t FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
return VERSION_202; return VERSION_202;
case VERSION_4_ONLY_FOR_TESTING: case VERSION_4_ONLY_FOR_TESTING:
return VERSION_4_ONLY_FOR_TESTING; return VERSION_4_ONLY_FOR_TESTING;
case VERSION_4: case VERSION_402:
return VERSION_4; return VERSION_402;
case VERSION_4_DEV: case VERSION_403:
return VERSION_4_DEV; return VERSION_403;
default: default:
return UNKNOWN_VERSION; return UNKNOWN_VERSION;
} }

View File

@ -38,8 +38,8 @@ class FormatUtils {
VERSION_201 = 201, VERSION_201 = 201,
VERSION_202 = 202, VERSION_202 = 202,
VERSION_4_ONLY_FOR_TESTING = 399, VERSION_4_ONLY_FOR_TESTING = 399,
VERSION_4 = 402, VERSION_402 = 402,
VERSION_4_DEV = 403, VERSION_403 = 403,
UNKNOWN_VERSION = -1 UNKNOWN_VERSION = -1
}; };

View File

@ -62,14 +62,14 @@ TEST(FormatUtilsTest, TestDetectFormatVersion) {
} }
{ {
const std::vector<uint8_t> buffer = const std::vector<uint8_t> buffer =
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_4, 0, 0); getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_402, 0, 0);
EXPECT_EQ(FormatUtils::VERSION_4, FormatUtils::detectFormatVersion( EXPECT_EQ(FormatUtils::VERSION_402, FormatUtils::detectFormatVersion(
ReadOnlyByteArrayView(buffer.data(), buffer.size()))); ReadOnlyByteArrayView(buffer.data(), buffer.size())));
} }
{ {
const std::vector<uint8_t> buffer = const std::vector<uint8_t> buffer =
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_4_DEV, 0, 0); getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_403, 0, 0);
EXPECT_EQ(FormatUtils::VERSION_4_DEV, FormatUtils::detectFormatVersion( EXPECT_EQ(FormatUtils::VERSION_403, FormatUtils::detectFormatVersion(
ReadOnlyByteArrayView(buffer.data(), buffer.size()))); ReadOnlyByteArrayView(buffer.data(), buffer.size())));
} }

View File

@ -49,7 +49,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private static final String TEST_LOCALE = "test"; private static final String TEST_LOCALE = "test";
private static final int DUMMY_PROBABILITY = 0; private static final int DUMMY_PROBABILITY = 0;
private static final int[] DICT_FORMAT_VERSIONS = private static final int[] DICT_FORMAT_VERSIONS =
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary"; private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
private int mCurrentTime = 0; private int mCurrentTime = 0;
@ -73,11 +73,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
} }
private static boolean supportsCountBasedNgram(final int formatVersion) { private static boolean supportsCountBasedNgram(final int formatVersion) {
return formatVersion >= FormatSpec.VERSION4_DEV; return formatVersion >= FormatSpec.VERSION403;
} }
private static boolean supportsNgram(final int formatVersion) { private static boolean supportsNgram(final int formatVersion) {
return formatVersion >= FormatSpec.VERSION4_DEV; return formatVersion >= FormatSpec.VERSION403;
} }
private void onInputWord(final BinaryDictionary binaryDictionary, final String word, private void onInputWord(final BinaryDictionary binaryDictionary, final String word,

View File

@ -45,19 +45,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final String TEST_LOCALE = "test"; private static final String TEST_LOCALE = "test";
private static final int[] DICT_FORMAT_VERSIONS = private static final int[] DICT_FORMAT_VERSIONS =
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
private static final String DICTIONARY_ID = "TestBinaryDictionary"; private static final String DICTIONARY_ID = "TestBinaryDictionary";
private static boolean canCheckBigramProbability(final int formatVersion) {
return formatVersion > FormatSpec.VERSION401;
}
private static boolean supportsBeginningOfSentence(final int formatVersion) {
return formatVersion > FormatSpec.VERSION401;
}
private static boolean supportsNgram(final int formatVersion) { private static boolean supportsNgram(final int formatVersion) {
return formatVersion >= FormatSpec.VERSION4_DEV; return formatVersion >= FormatSpec.VERSION403;
} }
private HashSet<File> mDictFilesToBeDeleted = new HashSet<>(); private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
@ -84,19 +76,13 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private File createEmptyDictionaryWithAttributesAndGetFile(final int formatVersion, private File createEmptyDictionaryWithAttributesAndGetFile(final int formatVersion,
final HashMap<String, String> attributeMap) { final HashMap<String, String> attributeMap) {
if (formatVersion == FormatSpec.VERSION4 try {
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion,
|| formatVersion == FormatSpec.VERSION4_DEV) { attributeMap);
try { mDictFilesToBeDeleted.add(dictFile);
final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion, return dictFile;
attributeMap); } catch (final IOException e) {
mDictFilesToBeDeleted.add(dictFile); fail(e.toString());
return dictFile;
} catch (final IOException e) {
fail(e.toString());
}
} else {
fail("Dictionary format version " + formatVersion + " is not supported.");
} }
return null; return null;
} }
@ -350,18 +336,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
assertTrue(isValidBigram(binaryDictionary, "abb", "aaa")); assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc")); assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
if (canCheckBigramProbability(formatVersion)) { assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
}
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) { assertEquals(updatedBigramProbability,
assertEquals(updatedBigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
getBigramProbability(binaryDictionary, "aaa", "abb"));
}
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
@ -381,17 +363,12 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, "abc", unigramProbability); addUnigramWord(binaryDictionary, "abc", unigramProbability);
addUnigramWord(binaryDictionary, "f", unigramProbability); addUnigramWord(binaryDictionary, "f", unigramProbability);
if (canCheckBigramProbability(formatVersion)) { assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abcde", "fghij"));
assertEquals(bigramProbability,
getBigramProbability(binaryDictionary, "abcde", "fghij"));
}
assertEquals(Dictionary.NOT_A_PROBABILITY, assertEquals(Dictionary.NOT_A_PROBABILITY,
getBigramProbability(binaryDictionary, "abcde", "fgh")); getBigramProbability(binaryDictionary, "abcde", "fgh"));
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
if (canCheckBigramProbability(formatVersion)) { assertEquals(updatedBigramProbability,
assertEquals(updatedBigramProbability, getBigramProbability(binaryDictionary, "abcde", "fghij"));
getBigramProbability(binaryDictionary, "abcde", "fghij"));
}
} }
public void testRandomlyAddBigramWords() { public void testRandomlyAddBigramWords() {
@ -441,10 +418,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = bigramProbabilities.get(bigram); final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) { assertEquals(bigramProbability,
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, bigram.first, bigram.second));
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
}
} }
} }
@ -594,12 +569,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
if (canCheckBigramProbability(formatVersion)) { assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
}
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa")); assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
@ -661,10 +634,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = bigramProbabilities.get(bigram); final int bigramProbability = bigramProbabilities.get(bigram);
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
if (canCheckBigramProbability(formatVersion)) { assertEquals(bigramProbability,
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, bigram.first, bigram.second));
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
}
} }
} }
@ -768,10 +739,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
probability = Dictionary.NOT_A_PROBABILITY; probability = Dictionary.NOT_A_PROBABILITY;
} }
if (canCheckBigramProbability(formatVersion)) { assertEquals(probability,
assertEquals(probability, getBigramProbability(binaryDictionary, bigram.first, bigram.second));
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
}
assertEquals(probability != Dictionary.NOT_A_PROBABILITY, assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
isValidBigram(binaryDictionary, bigram.first, bigram.second)); isValidBigram(binaryDictionary, bigram.first, bigram.second));
} }
@ -971,10 +940,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final WeightedString bigramTarget : wordProperty.getBigrams()) { for (final WeightedString bigramTarget : wordProperty.getBigrams()) {
final String word1 = bigramTarget.mWord; final String word1 = bigramTarget.mWord;
assertTrue(bigramWord1s.contains(word1)); assertTrue(bigramWord1s.contains(word1));
if (canCheckBigramProbability(formatVersion)) { final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1));
final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1)); assertEquals(bigramProbability, bigramTarget.getProbability());
assertEquals(bigramProbability, bigramTarget.getProbability());
}
} }
} }
} }
@ -1057,10 +1024,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final String word1 = bigramTarget.mWord; final String word1 = bigramTarget.mWord;
assertTrue(bigramWord1s.contains(word1)); assertTrue(bigramWord1s.contains(word1));
final Pair<String, String> bigram = new Pair<>(word0, word1); final Pair<String, String> bigram = new Pair<>(word0, word1);
if (canCheckBigramProbability(formatVersion)) { final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram);
final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram); assertEquals(bigramProbability, bigramTarget.getProbability());
assertEquals(bigramProbability, bigramTarget.getProbability());
}
bigramSet.remove(bigram); bigramSet.remove(bigram);
} }
} }
@ -1198,7 +1163,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
public void testPossiblyOffensiveAttributeMaintained() { public void testPossiblyOffensiveAttributeMaintained() {
final BinaryDictionary binaryDictionary = final BinaryDictionary binaryDictionary =
getEmptyBinaryDictionary(FormatSpec.VERSION4_DEV); getEmptyBinaryDictionary(FormatSpec.VERSION403);
binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY, binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY,
false, true, true, 0); false, true, true, 0);
WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false); WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false);
@ -1236,11 +1201,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
if (canCheckBigramProbability(toFormatVersion)) { assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb"));
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb")); assertEquals(bigramProbability, binaryDictionary.getNgramProbability(
assertEquals(bigramProbability, binaryDictionary.getNgramProbability( NgramContext.BEGINNING_OF_SENTENCE, "aaa"));
NgramContext.BEGINNING_OF_SENTENCE, "aaa"));
}
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc", WordProperty wordProperty = binaryDictionary.getWordProperty("ccc",
false /* isBeginningOfSentence */); false /* isBeginningOfSentence */);
@ -1311,10 +1274,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
binaryDictionary.getPropertyForGettingStats(BinaryDictionary.UNIGRAM_COUNT_QUERY))); binaryDictionary.getPropertyForGettingStats(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
for (final Pair<String, String> bigram : bigrams) { for (final Pair<String, String> bigram : bigrams) {
if (canCheckBigramProbability(toFormatVersion)) { assertEquals((int)bigramProbabilities.get(bigram),
assertEquals((int)bigramProbabilities.get(bigram), getBigramProbability(binaryDictionary, bigram.first, bigram.second));
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
}
assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second)); assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
} }
assertEquals(bigramProbabilities.size(), Integer.parseInt( assertEquals(bigramProbabilities.size(), Integer.parseInt(
@ -1323,9 +1284,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
public void testBeginningOfSentence() { public void testBeginningOfSentence() {
for (final int formatVersion : DICT_FORMAT_VERSIONS) { for (final int formatVersion : DICT_FORMAT_VERSIONS) {
if (supportsBeginningOfSentence(formatVersion)) { testBeginningOfSentence(formatVersion);
testBeginningOfSentence(formatVersion);
}
} }
} }