am 109ba3ac: Merge "Regenerate old version dictionaries using new format."

* commit '109ba3ace3815a7650a9fb0485b187968ddf8c29':
  Regenerate old version dictionaries using new format.

java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java

@@ -224,14 +224,10 @@ final public class BinaryDictionaryGetter {
         }
     }
 
-    // ## HACK ## we prevent usage of a dictionary before version 18 for English only. The reason
+    // ## HACK ## we prevent usage of a dictionary before version 18. The reason for this is, since
-    // for this is, since those do not include whitelist entries, the new code with an old version
+    // those do not include whitelist entries, the new code with an old version of the dictionary
-    // of the dictionary would lose whitelist functionality.
+    // would lose whitelist functionality.
     private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
-        // Only for English - other languages didn't have a whitelist, hence this
-        // ad-hoc ## HACK ##
-        if (!Locale.ENGLISH.getLanguage().equals(locale.getLanguage())) return true;
-
         FileInputStream inStream = null;
         try {
             // Read the version of the file

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -92,7 +92,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /** Controls access to the local binary dictionary for this instance. */
     private final DictionaryController mLocalDictionaryController = new DictionaryController();
 
-    private static final int BINARY_DICT_VERSION = 1;
+    // TODO: Regenerate version 3 binary dictionary.
+    private static final int BINARY_DICT_VERSION = 2;
     private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
             new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
 
@@ -415,6 +416,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                 // shared dictionary.
                 loadBinaryDictionary();
             }
+            if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) {
+                // Binary dictionary is not valid. Regenerate the dictionary file.
+                mSharedDictionaryController.mLastUpdateTime = time;
+                generateBinaryDictionary();
+                loadBinaryDictionary();
+            }
             mLocalDictionaryController.mLastUpdateTime = time;
         } finally {
             mSharedDictionaryController.unlock();

native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp

@@ -27,10 +27,6 @@ const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4;
 /**
  * Format versions
  */
-// Originally, format version 1 had a 16-bit magic number, then the version number `01'
-// then options that must be 0. Hence the first 32-bits of the format are always as follow
-// and it's okay to consider them a magic number as a whole.
-const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100;
 
 // The versions of Latin IME that only handle format version 1 only test for the magic
 // number, so we had to change it so that version 2 files would be rejected by older
@@ -50,12 +46,6 @@ const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12;
     }
     const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0);
     switch (magicNumber) {
-    case FORMAT_VERSION_1_MAGIC_NUMBER:
-        // Format 1 header is exactly 5 bytes long and looks like:
-        // Magic number (2 bytes) 0x78 0xB1
-        // Version number (1 byte) 0x01
-        // Options (2 bytes) must be 0x00 0x00
-        return VERSION_1;
     case FORMAT_VERSION_2_MAGIC_NUMBER:
         // Version 2 dictionaries are at least 12 bytes long.
         // If this dictionary has the version 2 magic number but is less than 12 bytes long,

native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h

@@ -33,10 +33,9 @@ namespace latinime {
  */
 class BinaryDictionaryFormatUtils {
  public:
-    // TODO: Remove obsolete version logic
+    // TODO: Support version 3 format.
     enum FORMAT_VERSION {
-        VERSION_1,
+        VERSION_2 = 1,
-        VERSION_2,
         UNKNOWN_VERSION
     };
 
@@ -46,7 +45,6 @@ class BinaryDictionaryFormatUtils {
     DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils);
 
     static const int DICTIONARY_MINIMUM_SIZE;
-    static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER;
     static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER;
     static const int FORMAT_VERSION_2_MINIMUM_SIZE;
 };

native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp

@@ -26,8 +26,6 @@ namespace latinime {
 
 const int BinaryDictionaryHeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256;
 
-const int BinaryDictionaryHeaderReadingUtils::FORMAT_VERSION_1_HEADER_SIZE = 5;
-
 const int BinaryDictionaryHeaderReadingUtils::VERSION_2_MAGIC_NUMBER_SIZE = 4;
 const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_VERSION_SIZE = 2;
 const int BinaryDictionaryHeaderReadingUtils::VERSION_2_DICTIONARY_FLAG_SIZE = 2;
@@ -48,8 +46,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
 /* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(
         const BinaryDictionaryInfo *const binaryDictionaryInfo) {
     switch (binaryDictionaryInfo->getFormat()) {
-        case BinaryDictionaryFormatUtils::VERSION_1:
-            return FORMAT_VERSION_1_HEADER_SIZE;
         case BinaryDictionaryFormatUtils::VERSION_2:
             // See the format of the header in the comment in
             // BinaryDictionaryFormatUtils::detectFormatVersion()
@@ -65,8 +61,6 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
         BinaryDictionaryHeaderReadingUtils::getFlags(
                 const BinaryDictionaryInfo *const binaryDictionaryInfo) {
     switch (binaryDictionaryInfo->getFormat()) {
-        case BinaryDictionaryFormatUtils::VERSION_1:
-            return NO_FLAGS;
         case BinaryDictionaryFormatUtils::VERSION_2:
             return ByteArrayUtils::readUint16(binaryDictionaryInfo->getDictBuf(),
                     VERSION_2_MAGIC_NUMBER_SIZE + VERSION_2_DICTIONARY_VERSION_SIZE);

native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.h

@@ -82,8 +82,6 @@ class BinaryDictionaryHeaderReadingUtils {
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryHeaderReadingUtils);
 
-    static const int FORMAT_VERSION_1_HEADER_SIZE;
-
     static const int VERSION_2_MAGIC_NUMBER_SIZE;
     static const int VERSION_2_DICTIONARY_VERSION_SIZE;
     static const int VERSION_2_DICTIONARY_FLAG_SIZE;