From a62b5b22eff2c1842fe1e0a4ea949e1e004de40b Mon Sep 17 00:00:00 2001 From: Keisuke Kuroynagi Date: Mon, 1 Jul 2013 15:25:33 +0900 Subject: [PATCH] Regenerate old version dictionaries using new format. Bug: 9617181 Change-Id: I228a3eac959f81b38e4092015e8e3c429d46fa60 --- .../inputmethod/latin/BinaryDictionaryGetter.java | 10 +++------- .../inputmethod/latin/ExpandableBinaryDictionary.java | 9 ++++++++- .../core/dictionary/binary_dictionary_format_utils.cpp | 10 ---------- .../core/dictionary/binary_dictionary_format_utils.h | 6 ++---- .../binary_dictionary_header_reading_utils.cpp | 6 ------ .../binary_dictionary_header_reading_utils.h | 2 -- 6 files changed, 13 insertions(+), 30 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 2cd3a26d8..2e1e66140 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -224,14 +224,10 @@ final public class BinaryDictionaryGetter { } } - // ## HACK ## we prevent usage of a dictionary before version 18 for English only. The reason - // for this is, since those do not include whitelist entries, the new code with an old version - // of the dictionary would lose whitelist functionality. + // ## HACK ## we prevent usage of a dictionary before version 18. The reason for this is, since + // those do not include whitelist entries, the new code with an old version of the dictionary + // would lose whitelist functionality. private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { - // Only for English - other languages didn't have a whitelist, hence this - // ad-hoc ## HACK ## - if (!Locale.ENGLISH.getLanguage().equals(locale.getLanguage())) return true; - FileInputStream inStream = null; try { // Read the version of the file diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 9cdb86c2d..a19363d54 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -92,7 +92,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Controls access to the local binary dictionary for this instance. */ private final DictionaryController mLocalDictionaryController = new DictionaryController(); - private static final int BINARY_DICT_VERSION = 1; + // TODO: Regenerate version 3 binary dictionary. + private static final int BINARY_DICT_VERSION = 2; private static final FormatSpec.FormatOptions FORMAT_OPTIONS = new FormatSpec.FormatOptions(BINARY_DICT_VERSION); @@ -415,6 +416,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // shared dictionary. loadBinaryDictionary(); } + if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { + // Binary dictionary is not valid. Regenerate the dictionary file. + mSharedDictionaryController.mLastUpdateTime = time; + generateBinaryDictionary(); + loadBinaryDictionary(); + } mLocalDictionaryController.mLastUpdateTime = time; } finally { mSharedDictionaryController.unlock(); diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp index bbb4ca3f0..f48386bba 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp @@ -27,10 +27,6 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; /** * Format versions */ -// Originally, format version 1 had a 16-bit magic number, then the version number `01' -// then options that must be 0. Hence the first 32-bits of the format are always as follow -// and it's okay to consider them a magic number as a whole. -const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; // The versions of Latin IME that only handle format version 1 only test for the magic // number, so we had to change it so that version 2 files would be rejected by older @@ -50,12 +46,6 @@ const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12; } const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0); switch (magicNumber) { - case FORMAT_VERSION_1_MAGIC_NUMBER: - // Format 1 header is exactly 5 bytes long and looks like: - // Magic number (2 bytes) 0x78 0xB1 - // Version number (1 byte) 0x01 - // Options (2 bytes) must be 0x00 0x00 - return VERSION_1; case FORMAT_VERSION_2_MAGIC_NUMBER: // Version 2 dictionaries are at least 12 bytes long. // If this dictionary has the version 2 magic number but is less than 12 bytes long, diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h index 33618b9f0..80067b255 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h @@ -33,10 +33,9 @@ namespace latinime { */ class BinaryDictionaryFormatUtils { public: - // TODO: Remove obsolete version logic + // TODO: Support version 3 format. enum FORMAT_VERSION { - VERSION_1, - VERSION_2, + VERSION_2 = 1, UNKNOWN_VERSION }; @@ -46,7 +45,6 @@ class BinaryDictionaryFormatUtils { DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils); static const int DICTIONARY_MINIMUM_SIZE; - static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER; static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER; static const int FORMAT_VERSION_2_MINIMUM_SIZE; }; diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp index 2c9593144..6e1b15ce7 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp @@ -26,8 +26,6 @@ namespace latinime { const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256; -const int BinaryDictionaryHeaderReadingUtils::FORMAT_VERSION_1_HEADER_SIZE = 5; - const int BinaryDictionaryHeaderReadingUtils::VERSION_2_MAGIC_NUMBER_SIZE = 4; const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_VERSION_SIZE = 2; const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_FLAG_SIZE = 2; @@ -48,8 +46,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags /* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize( const BinaryDictionaryInfo *const binaryDictionaryInfo) { switch (binaryDictionaryInfo->getFormat()) { - case BinaryDictionaryFormatUtils::VERSION_1: - return FORMAT_VERSION_1_HEADER_SIZE; case BinaryDictionaryFormatUtils::VERSION_2: // See the format of the header in the comment in // BinaryDictionaryFormatUtils::detectFormatVersion() @@ -65,8 +61,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags BinaryDictionaryHeaderReadingUtils::getFlags( const BinaryDictionaryInfo *const binaryDictionaryInfo) { switch (binaryDictionaryInfo->getFormat()) { - case BinaryDictionaryFormatUtils::VERSION_1: - return NO_FLAGS; case BinaryDictionaryFormatUtils::VERSION_2: return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(), VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE); diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h index 49ed2b9cc..94b9e124d 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h @@ -82,8 +82,6 @@ class BinaryDictionaryHeaderReadingUtils { private: DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils); - static const int FORMAT_VERSION_1_HEADER_SIZE; - static const int VERSION_2_MAGIC_NUMBER_SIZE; static const int VERSION_2_DICTIONARY_VERSION_SIZE; static const int VERSION_2_DICTIONARY_FLAG_SIZE;