Add VERSION4_TEST for testing.

Bug: 13406708
Change-Id: I0b434717ea3edbb131650af634c53cfbfe113c3e
main
Keisuke Kuroyanagi 2014-05-09 15:17:01 +09:00
parent 8a9d9de72a
commit 455dc84cf2
13 changed files with 102 additions and 24 deletions

View File

@ -541,7 +541,9 @@ public final class BinaryDictionary extends Dictionary {
close(); close();
final File dictFile = new File(mDictFilePath); final File dictFile = new File(mDictFilePath);
final File tmpDictFile = new File(tmpDictFilePath); final File tmpDictFile = new File(tmpDictFilePath);
FileUtils.deleteRecursively(dictFile); if (!FileUtils.deleteRecursively(dictFile)) {
return false;
}
if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) {
return false; return false;
} }

View File

@ -119,8 +119,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
} }
private boolean needsToMigrateDictionary(final int formatVersion) { private boolean needsToMigrateDictionary(final int formatVersion) {
// TODO: Check version. // When we bump up the dictionary format version, the old version should be added to here
return false; // for supporting migration. Note that native code has to support reading such formats.
return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING;
} }
public boolean isValidDictionaryLocked() { public boolean isValidDictionaryLocked() {

View File

@ -186,7 +186,12 @@ public final class FormatSpec {
// From version 4 on, we use version * 100 + revision as a version number. That allows // From version 4 on, we use version * 100 + revision as a version number. That allows
// us to change the format during development while having testing devices remove // us to change the format during development while having testing devices remove
// older files with each upgrade, while still having a readable versioning scheme. // older files with each upgrade, while still having a readable versioning scheme.
// When we bump up the dictionary format version, we should update
// ExpandableDictionary.needsToMigrateDictionary() and
// ExpandableDictionary.matchesExpectedBinaryDictFormatVersionForThisType().
public static final int VERSION2 = 2; public static final int VERSION2 = 2;
// Dictionary version used for testing.
public static final int VERSION4_ONLY_FOR_TESTING = 399;
public static final int VERSION4 = 401; public static final int VERSION4 = 401;
static final int MINIMUM_SUPPORTED_VERSION = VERSION2; static final int MINIMUM_SUPPORTED_VERSION = VERSION2;
static final int MAXIMUM_SUPPORTED_VERSION = VERSION4; static final int MAXIMUM_SUPPORTED_VERSION = VERSION4;

View File

@ -139,6 +139,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
switch (mDictFormatVersion) { switch (mDictFormatVersion) {
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
return FormatUtils::VERSION_2; return FormatUtils::VERSION_2;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
return FormatUtils::VERSION_4_ONLY_FOR_TESTING;
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
return FormatUtils::VERSION_4; return FormatUtils::VERSION_4;
default: default:

View File

@ -98,8 +98,9 @@ typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
// Version 2 dictionary writing is not supported. // Version 2 dictionary writing is not supported.
return false; return false;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */, return buffer->writeUintAndAdvancePosition(version /* data */,
HEADER_DICTIONARY_VERSION_SIZE, writingPos); HEADER_DICTIONARY_VERSION_SIZE, writingPos);
default: default:
return false; return false;

View File

@ -52,9 +52,11 @@ namespace latinime {
DictionaryStructureWithBufferPolicyFactory:: newPolicyForOnMemoryDict( DictionaryStructureWithBufferPolicyFactory:: newPolicyForOnMemoryDict(
const int formatVersion, const std::vector<int> &locale, const int formatVersion, const std::vector<int> &locale,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
switch (formatVersion) { FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
switch (dictFormatVersion) {
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: { case FormatUtils::VERSION_4: {
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, locale, attributeMap); HeaderPolicy headerPolicy(dictFormatVersion, locale, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy, Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE); Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE);
@ -87,11 +89,13 @@ namespace latinime {
if (!mmappedBuffer) { if (!mmappedBuffer) {
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
} }
switch (FormatUtils::detectFormatVersion(mmappedBuffer->getBuffer(), const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::detectFormatVersion(
mmappedBuffer->getBufferSize())) { mmappedBuffer->getBuffer(), mmappedBuffer->getBufferSize());
switch (formatVersion) {
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
AKLOGE("Given path is a directory but the format is version 2. path: %s", path); AKLOGE("Given path is a directory but the format is version 2. path: %s", path);
break; break;
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: { case FormatUtils::VERSION_4: {
const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */; const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */;
char dictPath[dictDirPathBufSize]; char dictPath[dictDirPathBufSize];
@ -102,7 +106,8 @@ namespace latinime {
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr);
} }
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer))); Ver4DictBuffers::openVer4DictBuffers(dictPath, std::move(mmappedBuffer),
formatVersion));
if (!dictBuffers || !dictBuffers->isValid()) { if (!dictBuffers || !dictBuffers->isValid()) {
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s", AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s",
path); path);
@ -135,6 +140,7 @@ namespace latinime {
case FormatUtils::VERSION_2: case FormatUtils::VERSION_2:
return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
new PatriciaTriePolicy(std::move(mmappedBuffer))); new PatriciaTriePolicy(std::move(mmappedBuffer)));
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
AKLOGE("Given path is a file but the format is version 4. path: %s", path); AKLOGE("Given path is a file but the format is version 4. path: %s", path);
break; break;

View File

@ -27,7 +27,8 @@
namespace latinime { namespace latinime {
/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer) { const char *const dictPath, MmappedBuffer::MmappedBufferPtr headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion) {
if (!headerBuffer) { if (!headerBuffer) {
ASSERT(false); ASSERT(false);
AKLOGE("The header buffer must be valid to open ver4 dict buffers."); AKLOGE("The header buffer must be valid to open ver4 dict buffers.");
@ -35,7 +36,8 @@ namespace latinime {
} }
// TODO: take only dictDirPath, and open both header and trie files in the constructor below // TODO: take only dictDirPath, and open both header and trie files in the constructor below
const bool isUpdatable = headerBuffer->isUpdatable(); const bool isUpdatable = headerBuffer->isUpdatable();
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable)); return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, std::move(headerBuffer), isUpdatable,
formatVersion));
} }
bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
@ -113,11 +115,12 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath,
} }
Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath,
MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable) MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion)
: mHeaderBuffer(std::move(headerBuffer)), : mHeaderBuffer(std::move(headerBuffer)),
mDictBuffer(MmappedBuffer::openBuffer(dictPath, mDictBuffer(MmappedBuffer::openBuffer(dictPath,
Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)),
mHeaderPolicy(mHeaderBuffer->getBuffer(), FormatUtils::VERSION_4), mHeaderPolicy(mHeaderBuffer->getBuffer(), formatVersion),
mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr, mExpandableHeaderBuffer(mHeaderBuffer ? mHeaderBuffer->getBuffer() : nullptr,
mHeaderPolicy.getSize(), mHeaderPolicy.getSize(),
BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),

View File

@ -36,7 +36,8 @@ class Ver4DictBuffers {
typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr; typedef std::unique_ptr<Ver4DictBuffers> Ver4DictBuffersPtr;
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
MmappedBuffer::MmappedBufferPtr headerBuffer); MmappedBuffer::MmappedBufferPtr headerBuffer,
const FormatUtils::FORMAT_VERSION formatVersion);
static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers(
const HeaderPolicy *const headerPolicy, const int maxTrieSize) { const HeaderPolicy *const headerPolicy, const int maxTrieSize) {
@ -120,7 +121,8 @@ class Ver4DictBuffers {
DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers);
Ver4DictBuffers(const char *const dictDirPath, Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable); const MmappedBuffer::MmappedBufferPtr headerBuffer, const bool isUpdatable,
const FormatUtils::FORMAT_VERSION formatVersion);
Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize); Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize);

View File

@ -34,9 +34,12 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
const int dictVersion, const std::vector<int> localeAsCodePointVector, const int dictVersion, const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
TimeKeeper::setCurrentTime(); TimeKeeper::setCurrentTime();
switch (dictVersion) { const FormatUtils::FORMAT_VERSION formatVersion = FormatUtils::getFormatVersion(dictVersion);
switch (formatVersion) {
case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
case FormatUtils::VERSION_4: case FormatUtils::VERSION_4:
return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap); return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap,
formatVersion);
default: default:
AKLOGE("Cannot create dictionary %s because format version %d is not supported.", AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
filePath, dictVersion); filePath, dictVersion);
@ -46,8 +49,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath, /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const std::vector<int> localeAsCodePointVector, const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap); const FormatUtils::FORMAT_VERSION formatVersion) {
HeaderPolicy headerPolicy(formatVersion, localeAsCodePointVector, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy, Ver4DictBuffers::createVer4DictBuffers(&headerPolicy,
Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE)); Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE));

View File

@ -21,6 +21,7 @@
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" #include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime { namespace latinime {
@ -46,7 +47,8 @@ class DictFileWritingUtils {
static bool createEmptyV4DictFile(const char *const filePath, static bool createEmptyV4DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector, const std::vector<int> localeAsCodePointVector,
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap); const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap,
const FormatUtils::FORMAT_VERSION formatVersion);
static bool flushBufferToFile(const char *const filePath, static bool flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer); const BufferWithExtendableBuffer *const buffer);

View File

@ -25,6 +25,18 @@ const uint32_t FormatUtils::MAGIC_NUMBER = 0x9BC13AFE;
// Magic number (4 bytes), version (2 bytes), flags (2 bytes), header size (4 bytes) = 12 // Magic number (4 bytes), version (2 bytes), flags (2 bytes), header size (4 bytes) = 12
const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
/* static */ FormatUtils::FORMAT_VERSION FormatUtils::getFormatVersion(const int formatVersion) {
switch (formatVersion) {
case VERSION_2:
return VERSION_2;
case VERSION_4_ONLY_FOR_TESTING:
return VERSION_4_ONLY_FOR_TESTING;
case VERSION_4:
return VERSION_4;
default:
return UNKNOWN_VERSION;
}
}
/* static */ FormatUtils::FORMAT_VERSION FormatUtils::detectFormatVersion( /* static */ FormatUtils::FORMAT_VERSION FormatUtils::detectFormatVersion(
const uint8_t *const dict, const int dictSize) { const uint8_t *const dict, const int dictSize) {
// The magic number is stored big-endian. // The magic number is stored big-endian.
@ -46,6 +58,8 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12;
// same so we use them for both here. // same so we use them for both here.
if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) { if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) {
return VERSION_2; return VERSION_2;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4_ONLY_FOR_TESTING) {
return VERSION_4_ONLY_FOR_TESTING;
} else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) {
return VERSION_4; return VERSION_4;
} else { } else {

View File

@ -31,6 +31,7 @@ class FormatUtils {
enum FORMAT_VERSION { enum FORMAT_VERSION {
// These MUST have the same values as the relevant constants in FormatSpec.java. // These MUST have the same values as the relevant constants in FormatSpec.java.
VERSION_2 = 2, VERSION_2 = 2,
VERSION_4_ONLY_FOR_TESTING = 399,
VERSION_4 = 401, VERSION_4 = 401,
UNKNOWN_VERSION = -1 UNKNOWN_VERSION = -1
}; };
@ -39,6 +40,7 @@ class FormatUtils {
// unsupported or obsolete dictionary formats. // unsupported or obsolete dictionary formats.
static const uint32_t MAGIC_NUMBER; static const uint32_t MAGIC_NUMBER;
static FORMAT_VERSION getFormatVersion(const int formatVersion);
static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize);
private: private:

View File

@ -46,21 +46,23 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private File createEmptyDictionaryAndGetFile(final String dictId, private File createEmptyDictionaryAndGetFile(final String dictId,
final int formatVersion) throws IOException { final int formatVersion) throws IOException {
if (formatVersion == FormatSpec.VERSION4) { if (formatVersion == FormatSpec.VERSION4
return createEmptyVer4DictionaryAndGetFile(dictId); || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING) {
return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion);
} else { } else {
throw new IOException("Dictionary format version " + formatVersion throw new IOException("Dictionary format version " + formatVersion
+ " is not supported."); + " is not supported.");
} }
} }
private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException { private File createEmptyVer4DictionaryAndGetFile(final String dictId,
final int formatVersion) throws IOException {
final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir()); getContext().getCacheDir());
file.delete(); file.delete();
file.mkdir(); file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>(); Map<String, String> attributeMap = new HashMap<String, String>();
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion,
Locale.ENGLISH, attributeMap)) { Locale.ENGLISH, attributeMap)) {
return file; return file;
} else { } else {
@ -1223,4 +1225,36 @@ public class BinaryDictionaryTests extends AndroidTestCase {
} }
} }
} }
public void testDictMigration() {
testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, FormatSpec.VERSION4);
}
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
File dictFile = null;
try {
dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion);
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
}
final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
final int unigramProbability = 100;
addUnigramWord(binaryDictionary, "aaa", unigramProbability);
addUnigramWord(binaryDictionary, "bbb", unigramProbability);
final int bigramProbability = 10;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion));
assertTrue(binaryDictionary.isValidDictionary());
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa"));
assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb"));
// TODO: Add tests for bigram frequency when the implementation gets ready.
assertTrue(binaryDictionary.isValidBigram("aaa", "bbb"));
}
} }