am b3bae2e8: Merge "Update v4 format version from 402 to 403."
* commit 'b3bae2e89bdd6046ff6cc1fd0fe4be889e66ec17': Update v4 format version from 402 to 403.main
commit
87af779f84
|
@ -120,7 +120,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
private static boolean needsToMigrateDictionary(final int formatVersion) {
|
private static boolean needsToMigrateDictionary(final int formatVersion) {
|
||||||
// When we bump up the dictionary format version, the old version should be added to here
|
// When we bump up the dictionary format version, the old version should be added to here
|
||||||
// for supporting migration. Note that native code has to support reading such formats.
|
// for supporting migration. Note that native code has to support reading such formats.
|
||||||
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING;
|
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
||||||
|
|| formatVersion == FormatSpec.VERSION402;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isValidDictionaryLocked() {
|
public boolean isValidDictionaryLocked() {
|
||||||
|
|
|
@ -175,9 +175,10 @@ public final class FormatSpec {
|
||||||
public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201;
|
public static final int MINIMUM_SUPPORTED_VERSION_OF_CODE_POINT_TABLE = VERSION201;
|
||||||
// Dictionary version used for testing.
|
// Dictionary version used for testing.
|
||||||
public static final int VERSION4_ONLY_FOR_TESTING = 399;
|
public static final int VERSION4_ONLY_FOR_TESTING = 399;
|
||||||
public static final int VERSION401 = 401;
|
public static final int VERSION402 = 402;
|
||||||
public static final int VERSION4 = 402;
|
public static final int VERSION403 = 403;
|
||||||
public static final int VERSION4_DEV = 403;
|
public static final int VERSION4 = VERSION403;
|
||||||
|
public static final int VERSION4_DEV = VERSION403;
|
||||||
static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
|
static final int MINIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
|
||||||
static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
|
static final int MAXIMUM_SUPPORTED_STATIC_VERSION = VERSION202;
|
||||||
static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4;
|
static final int MINIMUM_SUPPORTED_DYNAMIC_VERSION = VERSION4;
|
||||||
|
|
|
@ -141,10 +141,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
|
||||||
return FormatUtils::VERSION_202;
|
return FormatUtils::VERSION_202;
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
|
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
|
||||||
case FormatUtils::VERSION_4:
|
case FormatUtils::VERSION_402:
|
||||||
return FormatUtils::VERSION_4;
|
return FormatUtils::VERSION_402;
|
||||||
case FormatUtils::VERSION_4_DEV:
|
case FormatUtils::VERSION_403:
|
||||||
return FormatUtils::VERSION_4_DEV;
|
return FormatUtils::VERSION_403;
|
||||||
default:
|
default:
|
||||||
return FormatUtils::UNKNOWN_VERSION;
|
return FormatUtils::UNKNOWN_VERSION;
|
||||||
}
|
}
|
||||||
|
@ -247,7 +247,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool supportsBeginningOfSentence() const {
|
bool supportsBeginningOfSentence() const {
|
||||||
return mDictFormatVersion >= FormatUtils::VERSION_4;
|
return mDictFormatVersion >= FormatUtils::VERSION_402;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int *getCodePointTable() const {
|
const int *getCodePointTable() const {
|
||||||
|
|
|
@ -115,8 +115,8 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
|
||||||
// None of the static dictionaries (v2x) support writing
|
// None of the static dictionaries (v2x) support writing
|
||||||
return false;
|
return false;
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
case FormatUtils::VERSION_4:
|
case FormatUtils::VERSION_402:
|
||||||
case FormatUtils::VERSION_4_DEV:
|
case FormatUtils::VERSION_403:
|
||||||
return buffer->writeUintAndAdvancePosition(version /* data */,
|
return buffer->writeUintAndAdvancePosition(version /* data */,
|
||||||
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
|
HEADER_DICTIONARY_VERSION_SIZE, writingPos);
|
||||||
default:
|
default:
|
||||||
|
|
|
@ -58,7 +58,7 @@ namespace latinime {
|
||||||
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
|
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
|
||||||
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
|
FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
|
||||||
switch (dictFormatVersion) {
|
switch (dictFormatVersion) {
|
||||||
case FormatUtils::VERSION_4: {
|
case FormatUtils::VERSION_402: {
|
||||||
return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
|
return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
|
||||||
backward::v402::Ver4DictBuffers,
|
backward::v402::Ver4DictBuffers,
|
||||||
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
|
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
|
||||||
|
@ -66,7 +66,7 @@ namespace latinime {
|
||||||
dictFormatVersion, locale, attributeMap);
|
dictFormatVersion, locale, attributeMap);
|
||||||
}
|
}
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
case FormatUtils::VERSION_4_DEV: {
|
case FormatUtils::VERSION_403: {
|
||||||
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
|
return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
|
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
|
||||||
dictFormatVersion, locale, attributeMap);
|
dictFormatVersion, locale, attributeMap);
|
||||||
|
@ -118,7 +118,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
|
||||||
case FormatUtils::VERSION_202:
|
case FormatUtils::VERSION_202:
|
||||||
AKLOGE("Given path is a directory but the format is version 2xx. path: %s", path);
|
AKLOGE("Given path is a directory but the format is version 2xx. path: %s", path);
|
||||||
break;
|
break;
|
||||||
case FormatUtils::VERSION_4: {
|
case FormatUtils::VERSION_402: {
|
||||||
return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
|
return newPolicyForV4Dict<backward::v402::Ver4DictConstants,
|
||||||
backward::v402::Ver4DictBuffers,
|
backward::v402::Ver4DictBuffers,
|
||||||
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
|
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
|
||||||
|
@ -126,7 +126,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
|
||||||
headerFilePath, formatVersion, std::move(mmappedBuffer));
|
headerFilePath, formatVersion, std::move(mmappedBuffer));
|
||||||
}
|
}
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
case FormatUtils::VERSION_4_DEV: {
|
case FormatUtils::VERSION_403: {
|
||||||
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
|
return newPolicyForV4Dict<Ver4DictConstants, Ver4DictBuffers,
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
|
Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
|
||||||
headerFilePath, formatVersion, std::move(mmappedBuffer));
|
headerFilePath, formatVersion, std::move(mmappedBuffer));
|
||||||
|
@ -184,8 +184,8 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
|
||||||
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
|
||||||
new PatriciaTriePolicy(std::move(mmappedBuffer)));
|
new PatriciaTriePolicy(std::move(mmappedBuffer)));
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
case FormatUtils::VERSION_4:
|
case FormatUtils::VERSION_402:
|
||||||
case FormatUtils::VERSION_4_DEV:
|
case FormatUtils::VERSION_403:
|
||||||
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
|
AKLOGE("Given path is a file but the format is version 4. path: %s", path);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
|
|
|
@ -44,13 +44,13 @@ const int DictFileWritingUtils::SIZE_OF_BUFFER_SIZE_FIELD = 4;
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
|
const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
|
||||||
switch (formatVersion) {
|
switch (formatVersion) {
|
||||||
case FormatUtils::VERSION_4:
|
case FormatUtils::VERSION_402:
|
||||||
return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
|
return createEmptyV4DictFile<backward::v402::Ver4DictConstants,
|
||||||
backward::v402::Ver4DictBuffers,
|
backward::v402::Ver4DictBuffers,
|
||||||
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
|
backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr>(
|
||||||
filePath, localeAsCodePointVector, attributeMap, formatVersion);
|
filePath, localeAsCodePointVector, attributeMap, formatVersion);
|
||||||
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
|
||||||
case FormatUtils::VERSION_4_DEV:
|
case FormatUtils::VERSION_403:
|
||||||
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,
|
return createEmptyV4DictFile<Ver4DictConstants, Ver4DictBuffers,
|
||||||
Ver4DictBuffers::Ver4DictBuffersPtr>(
|
Ver4DictBuffers::Ver4DictBuffersPtr>(
|
||||||
filePath, localeAsCodePointVector, attributeMap, formatVersion);
|
filePath, localeAsCodePointVector, attributeMap, formatVersion);
|
||||||
|
|
|
@ -35,10 +35,10 @@ const size_t FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
|
||||||
return VERSION_202;
|
return VERSION_202;
|
||||||
case VERSION_4_ONLY_FOR_TESTING:
|
case VERSION_4_ONLY_FOR_TESTING:
|
||||||
return VERSION_4_ONLY_FOR_TESTING;
|
return VERSION_4_ONLY_FOR_TESTING;
|
||||||
case VERSION_4:
|
case VERSION_402:
|
||||||
return VERSION_4;
|
return VERSION_402;
|
||||||
case VERSION_4_DEV:
|
case VERSION_403:
|
||||||
return VERSION_4_DEV;
|
return VERSION_403;
|
||||||
default:
|
default:
|
||||||
return UNKNOWN_VERSION;
|
return UNKNOWN_VERSION;
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,8 +38,8 @@ class FormatUtils {
|
||||||
VERSION_201 = 201,
|
VERSION_201 = 201,
|
||||||
VERSION_202 = 202,
|
VERSION_202 = 202,
|
||||||
VERSION_4_ONLY_FOR_TESTING = 399,
|
VERSION_4_ONLY_FOR_TESTING = 399,
|
||||||
VERSION_4 = 402,
|
VERSION_402 = 402,
|
||||||
VERSION_4_DEV = 403,
|
VERSION_403 = 403,
|
||||||
UNKNOWN_VERSION = -1
|
UNKNOWN_VERSION = -1
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -62,14 +62,14 @@ TEST(FormatUtilsTest, TestDetectFormatVersion) {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
const std::vector<uint8_t> buffer =
|
const std::vector<uint8_t> buffer =
|
||||||
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_4, 0, 0);
|
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_402, 0, 0);
|
||||||
EXPECT_EQ(FormatUtils::VERSION_4, FormatUtils::detectFormatVersion(
|
EXPECT_EQ(FormatUtils::VERSION_402, FormatUtils::detectFormatVersion(
|
||||||
ReadOnlyByteArrayView(buffer.data(), buffer.size())));
|
ReadOnlyByteArrayView(buffer.data(), buffer.size())));
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
const std::vector<uint8_t> buffer =
|
const std::vector<uint8_t> buffer =
|
||||||
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_4_DEV, 0, 0);
|
getBuffer(FormatUtils::MAGIC_NUMBER, FormatUtils::VERSION_403, 0, 0);
|
||||||
EXPECT_EQ(FormatUtils::VERSION_4_DEV, FormatUtils::detectFormatVersion(
|
EXPECT_EQ(FormatUtils::VERSION_403, FormatUtils::detectFormatVersion(
|
||||||
ReadOnlyByteArrayView(buffer.data(), buffer.size())));
|
ReadOnlyByteArrayView(buffer.data(), buffer.size())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -49,7 +49,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
private static final String TEST_LOCALE = "test";
|
private static final String TEST_LOCALE = "test";
|
||||||
private static final int DUMMY_PROBABILITY = 0;
|
private static final int DUMMY_PROBABILITY = 0;
|
||||||
private static final int[] DICT_FORMAT_VERSIONS =
|
private static final int[] DICT_FORMAT_VERSIONS =
|
||||||
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
|
||||||
private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
|
private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
|
||||||
|
|
||||||
private int mCurrentTime = 0;
|
private int mCurrentTime = 0;
|
||||||
|
@ -73,11 +73,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean supportsCountBasedNgram(final int formatVersion) {
|
private static boolean supportsCountBasedNgram(final int formatVersion) {
|
||||||
return formatVersion >= FormatSpec.VERSION4_DEV;
|
return formatVersion >= FormatSpec.VERSION403;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean supportsNgram(final int formatVersion) {
|
private static boolean supportsNgram(final int formatVersion) {
|
||||||
return formatVersion >= FormatSpec.VERSION4_DEV;
|
return formatVersion >= FormatSpec.VERSION403;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void onInputWord(final BinaryDictionary binaryDictionary, final String word,
|
private void onInputWord(final BinaryDictionary binaryDictionary, final String word,
|
||||||
|
|
|
@ -45,19 +45,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
|
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
|
||||||
private static final String TEST_LOCALE = "test";
|
private static final String TEST_LOCALE = "test";
|
||||||
private static final int[] DICT_FORMAT_VERSIONS =
|
private static final int[] DICT_FORMAT_VERSIONS =
|
||||||
new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
|
new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
|
||||||
private static final String DICTIONARY_ID = "TestBinaryDictionary";
|
private static final String DICTIONARY_ID = "TestBinaryDictionary";
|
||||||
|
|
||||||
private static boolean canCheckBigramProbability(final int formatVersion) {
|
|
||||||
return formatVersion > FormatSpec.VERSION401;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean supportsBeginningOfSentence(final int formatVersion) {
|
|
||||||
return formatVersion > FormatSpec.VERSION401;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean supportsNgram(final int formatVersion) {
|
private static boolean supportsNgram(final int formatVersion) {
|
||||||
return formatVersion >= FormatSpec.VERSION4_DEV;
|
return formatVersion >= FormatSpec.VERSION403;
|
||||||
}
|
}
|
||||||
|
|
||||||
private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
|
private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
|
||||||
|
@ -84,19 +76,13 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
private File createEmptyDictionaryWithAttributesAndGetFile(final int formatVersion,
|
private File createEmptyDictionaryWithAttributesAndGetFile(final int formatVersion,
|
||||||
final HashMap<String, String> attributeMap) {
|
final HashMap<String, String> attributeMap) {
|
||||||
if (formatVersion == FormatSpec.VERSION4
|
try {
|
||||||
|| formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
|
final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion,
|
||||||
|| formatVersion == FormatSpec.VERSION4_DEV) {
|
attributeMap);
|
||||||
try {
|
mDictFilesToBeDeleted.add(dictFile);
|
||||||
final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion,
|
return dictFile;
|
||||||
attributeMap);
|
} catch (final IOException e) {
|
||||||
mDictFilesToBeDeleted.add(dictFile);
|
fail(e.toString());
|
||||||
return dictFile;
|
|
||||||
} catch (final IOException e) {
|
|
||||||
fail(e.toString());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
fail("Dictionary format version " + formatVersion + " is not supported.");
|
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -350,18 +336,14 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
|
assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc"));
|
||||||
assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
|
assertTrue(isValidBigram(binaryDictionary, "abb", "aaa"));
|
||||||
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
|
assertTrue(isValidBigram(binaryDictionary, "abb", "bcc"));
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
|
|
||||||
}
|
|
||||||
|
|
||||||
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
|
addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability);
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(updatedBigramProbability,
|
||||||
assertEquals(updatedBigramProbability,
|
getBigramProbability(binaryDictionary, "aaa", "abb"));
|
||||||
getBigramProbability(binaryDictionary, "aaa", "abb"));
|
|
||||||
}
|
|
||||||
|
|
||||||
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
|
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
|
||||||
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
|
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
|
||||||
|
@ -381,17 +363,12 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
addUnigramWord(binaryDictionary, "abc", unigramProbability);
|
addUnigramWord(binaryDictionary, "abc", unigramProbability);
|
||||||
addUnigramWord(binaryDictionary, "f", unigramProbability);
|
addUnigramWord(binaryDictionary, "f", unigramProbability);
|
||||||
|
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
||||||
assertEquals(bigramProbability,
|
|
||||||
getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
|
||||||
}
|
|
||||||
assertEquals(Dictionary.NOT_A_PROBABILITY,
|
assertEquals(Dictionary.NOT_A_PROBABILITY,
|
||||||
getBigramProbability(binaryDictionary, "abcde", "fgh"));
|
getBigramProbability(binaryDictionary, "abcde", "fgh"));
|
||||||
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
|
addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability);
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(updatedBigramProbability,
|
||||||
assertEquals(updatedBigramProbability,
|
getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
||||||
getBigramProbability(binaryDictionary, "abcde", "fghij"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRandomlyAddBigramWords() {
|
public void testRandomlyAddBigramWords() {
|
||||||
|
@ -441,10 +418,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int bigramProbability = bigramProbabilities.get(bigram);
|
final int bigramProbability = bigramProbabilities.get(bigram);
|
||||||
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
|
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
|
||||||
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(bigramProbability,
|
||||||
assertEquals(bigramProbability,
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -594,12 +569,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("abb"));
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc"));
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa"));
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc"));
|
|
||||||
}
|
|
||||||
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
|
assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa"));
|
||||||
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
|
assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc"));
|
||||||
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
|
assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa"));
|
||||||
|
@ -661,10 +634,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final int bigramProbability = bigramProbabilities.get(bigram);
|
final int bigramProbability = bigramProbabilities.get(bigram);
|
||||||
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
|
assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY,
|
||||||
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(bigramProbability,
|
||||||
assertEquals(bigramProbability,
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -768,10 +739,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
probability = Dictionary.NOT_A_PROBABILITY;
|
probability = Dictionary.NOT_A_PROBABILITY;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
assertEquals(probability,
|
||||||
assertEquals(probability,
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
|
||||||
}
|
|
||||||
assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
|
assertEquals(probability != Dictionary.NOT_A_PROBABILITY,
|
||||||
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
||||||
}
|
}
|
||||||
|
@ -971,10 +940,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
for (final WeightedString bigramTarget : wordProperty.getBigrams()) {
|
for (final WeightedString bigramTarget : wordProperty.getBigrams()) {
|
||||||
final String word1 = bigramTarget.mWord;
|
final String word1 = bigramTarget.mWord;
|
||||||
assertTrue(bigramWord1s.contains(word1));
|
assertTrue(bigramWord1s.contains(word1));
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1));
|
||||||
final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1));
|
assertEquals(bigramProbability, bigramTarget.getProbability());
|
||||||
assertEquals(bigramProbability, bigramTarget.getProbability());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1057,10 +1024,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
final String word1 = bigramTarget.mWord;
|
final String word1 = bigramTarget.mWord;
|
||||||
assertTrue(bigramWord1s.contains(word1));
|
assertTrue(bigramWord1s.contains(word1));
|
||||||
final Pair<String, String> bigram = new Pair<>(word0, word1);
|
final Pair<String, String> bigram = new Pair<>(word0, word1);
|
||||||
if (canCheckBigramProbability(formatVersion)) {
|
final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram);
|
||||||
final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram);
|
assertEquals(bigramProbability, bigramTarget.getProbability());
|
||||||
assertEquals(bigramProbability, bigramTarget.getProbability());
|
|
||||||
}
|
|
||||||
bigramSet.remove(bigram);
|
bigramSet.remove(bigram);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1198,7 +1163,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
public void testPossiblyOffensiveAttributeMaintained() {
|
public void testPossiblyOffensiveAttributeMaintained() {
|
||||||
final BinaryDictionary binaryDictionary =
|
final BinaryDictionary binaryDictionary =
|
||||||
getEmptyBinaryDictionary(FormatSpec.VERSION4_DEV);
|
getEmptyBinaryDictionary(FormatSpec.VERSION403);
|
||||||
binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY,
|
binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY,
|
||||||
false, true, true, 0);
|
false, true, true, 0);
|
||||||
WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false);
|
WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false);
|
||||||
|
@ -1236,11 +1201,9 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
|
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
|
||||||
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
|
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
|
||||||
if (canCheckBigramProbability(toFormatVersion)) {
|
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb"));
|
||||||
assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb"));
|
assertEquals(bigramProbability, binaryDictionary.getNgramProbability(
|
||||||
assertEquals(bigramProbability, binaryDictionary.getNgramProbability(
|
NgramContext.BEGINNING_OF_SENTENCE, "aaa"));
|
||||||
NgramContext.BEGINNING_OF_SENTENCE, "aaa"));
|
|
||||||
}
|
|
||||||
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
|
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
|
||||||
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc",
|
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc",
|
||||||
false /* isBeginningOfSentence */);
|
false /* isBeginningOfSentence */);
|
||||||
|
@ -1311,10 +1274,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
binaryDictionary.getPropertyForGettingStats(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
|
binaryDictionary.getPropertyForGettingStats(BinaryDictionary.UNIGRAM_COUNT_QUERY)));
|
||||||
|
|
||||||
for (final Pair<String, String> bigram : bigrams) {
|
for (final Pair<String, String> bigram : bigrams) {
|
||||||
if (canCheckBigramProbability(toFormatVersion)) {
|
assertEquals((int)bigramProbabilities.get(bigram),
|
||||||
assertEquals((int)bigramProbabilities.get(bigram),
|
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
||||||
getBigramProbability(binaryDictionary, bigram.first, bigram.second));
|
|
||||||
}
|
|
||||||
assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
|
||||||
}
|
}
|
||||||
assertEquals(bigramProbabilities.size(), Integer.parseInt(
|
assertEquals(bigramProbabilities.size(), Integer.parseInt(
|
||||||
|
@ -1323,9 +1284,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
|
||||||
|
|
||||||
public void testBeginningOfSentence() {
|
public void testBeginningOfSentence() {
|
||||||
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
|
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
|
||||||
if (supportsBeginningOfSentence(formatVersion)) {
|
testBeginningOfSentence(formatVersion);
|
||||||
testBeginningOfSentence(formatVersion);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue