Merge "[HD04] Make the locale mandatory."

java/src/com/android/inputmethod/latin/BinaryDictionary.java

@@ -139,7 +139,7 @@ public final class BinaryDictionary extends Dictionary {
     }
 
     private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
-            String[] attributeKeyStringArray, String[] attributeValueStringArray);
+            String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
     private static native long openNative(String sourceDir, long dictOffset, long dictSize,
             boolean isUpdatable);
     private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
@@ -179,7 +179,7 @@ public final class BinaryDictionary extends Dictionary {
     private static native String getPropertyNative(long dict, String query);
 
     public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
-            final Map<String, String> attributeMap) {
+            final Locale locale, final Map<String, String> attributeMap) {
         final String[] keyArray = new String[attributeMap.size()];
         final String[] valueArray = new String[attributeMap.size()];
         int index = 0;
@@ -188,7 +188,8 @@ public final class BinaryDictionary extends Dictionary {
             valueArray[index] = attributeMap.get(key);
             index++;
         }
-        return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
+        return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
+                valueArray);
     }
 
     // TODO: Move native dict into session

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -289,10 +289,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                         Log.e(TAG, "Can't remove a file: " + file.getName());
                     }
                     BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
-                            DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
+                            DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
                     mBinaryDictionary = new BinaryDictionary(
                             file.getAbsolutePath(), 0 /* offset */, file.length(),
-                            true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
+                            true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
                 } else {
                     mDictionaryWriter.clear();
                 }
@@ -594,7 +594,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                     Log.e(TAG, "Can't remove a file: " + file.getName());
                 }
                 BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
-                        DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
+                        DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
             } else {
                 if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
                     mBinaryDictionary.flushWithGC();

java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java

@@ -55,7 +55,9 @@ public class Ver4DictEncoder implements DictEncoder {
             throw new UnsupportedFormatException("Given path is not a directory.");
         }
         if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
-                FormatSpec.VERSION4, dict.mOptions.mAttributes)) {
+                FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString(
+                dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)),
+                dict.mOptions.mAttributes)) {
             throw new IOException("Cannot create dictionary file : "
                 + mDictPlacedDir.getAbsolutePath());
         }

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

@@ -29,6 +29,7 @@
 #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
 #include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
 #include "utils/autocorrection_threshold_utils.h"
+#include "utils/char_utils.h"
 #include "utils/time_keeper.h"
 
 namespace latinime {
@@ -37,13 +38,15 @@ class ProximityInfo;
 
 // TODO: Move to makedict.
 static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
-        jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
+        jstring filePath, jlong dictVersion, jstring locale, jobjectArray attributeKeyStringArray,
         jobjectArray attributeValueStringArray) {
     const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
     char filePathChars[filePathUtf8Length + 1];
     env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
     filePathChars[filePathUtf8Length] = '\0';
-
+    jsize localeLength = env->GetStringLength(locale);
+    jchar localeCodePoints[localeLength];
+    env->GetStringRegion(locale, 0, localeLength, localeCodePoints);
     const int keyCount = env->GetArrayLength(attributeKeyStringArray);
     const int valueCount = env->GetArrayLength(attributeValueStringArray);
     if (keyCount != valueCount) {
@@ -73,7 +76,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
     }
 
     return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
-            &attributeMap);
+            CharUtils::convertShortArrayToIntVector(localeCodePoints, localeLength), &attributeMap);
 }
 
 static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
@@ -503,7 +506,8 @@ static int latinime_BinaryDictionary_setCurrentTimeForTest(JNIEnv *env, jclass c
 static const JNINativeMethod sMethods[] = {
     {
         const_cast<char *>("createEmptyDictFileNative"),
-        const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
+        const_cast<char *>(
+                "(Ljava/lang/String;JLjava/lang/String;[Ljava/lang/String;[Ljava/lang/String;)Z"),
         reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
     },
     {

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp

@@ -32,6 +32,7 @@ const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE
 // Historical info is information that is needed to support decaying such as timestamp, level and
 // count.
 const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
+const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration
 const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
 const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
 
@@ -59,6 +60,10 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
     outValue[terminalIndex] = '\0';
 }
 
+const std::vector<int> HeaderPolicy::readLocale() const {
+    return HeaderReadWriteUtils::readCodePointVectorAttributeValue(&mAttributeMap, LOCALE_KEY);
+}
+
 float HeaderPolicy::readMultipleWordCostMultiplier() const {
     const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
             MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
@@ -116,6 +121,7 @@ void HeaderPolicy::fillInHeader(const bool updatesLastDecayedTime, const int uni
     // Set the current time as the generation time.
     HeaderReadWriteUtils::setIntAttribute(outAttributeMap, DATE_KEY,
             TimeKeeper::peekCurrentTime());
+    HeaderReadWriteUtils::setCodePointVectorAttribute(outAttributeMap, LOCALE_KEY, mLocale);
     if (updatesLastDecayedTime) {
         // Set current time as the last updated time.
         HeaderReadWriteUtils::setIntAttribute(outAttributeMap, LAST_DECAYED_TIME_KEY,

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

@@ -23,6 +23,7 @@
 #include "suggest/core/policy/dictionary_header_structure_policy.h"
 #include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
 #include "suggest/policyimpl/dictionary/utils/format_utils.h"
+#include "utils/char_utils.h"
 #include "utils/time_keeper.h"
 
 namespace latinime {
@@ -35,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
               mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
               mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
               mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
+              mLocale(readLocale()),
               mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
               mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
               mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@@ -54,10 +56,11 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
 
     // Constructs header information using an attribute map.
     HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
+            const std::vector<int> locale,
             const HeaderReadWriteUtils::AttributeMap *const attributeMap)
             : mDictFormatVersion(dictFormatVersion),
               mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
-                      attributeMap)), mSize(0), mAttributeMap(*attributeMap),
+                      attributeMap)), mSize(0), mAttributeMap(*attributeMap), mLocale(locale),
               mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
               mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
               mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@@ -68,12 +71,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
                       DATE_KEY, TimeKeeper::peekCurrentTime() /* defaultValue */)),
               mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0),
               mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
-                      &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
+                      &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {
+        }
 
     // Temporary dummy header.
     HeaderPolicy()
             : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
-              mAttributeMap(), mMultiWordCostMultiplier(0.0f),
+              mAttributeMap(), mLocale(CharUtils::EMPTY_STRING), mMultiWordCostMultiplier(0.0f),
               mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
               mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
               mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
@@ -174,6 +178,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     static const char *const BIGRAM_COUNT_KEY;
     static const char *const EXTENDED_REGION_SIZE_KEY;
     static const char *const HAS_HISTORICAL_INFO_KEY;
+    static const char *const LOCALE_KEY;
     static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
     static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
 
@@ -181,6 +186,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
     const int mSize;
     HeaderReadWriteUtils::AttributeMap mAttributeMap;
+    const std::vector<int> mLocale;
     const float mMultiWordCostMultiplier;
     const bool mRequiresGermanUmlautProcessing;
     const bool mIsDecayingDict;
@@ -191,6 +197,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     const int mExtendedRegionSize;
     const bool mHasHistoricalInfoOfWords;
 
+    const std::vector<int> readLocale() const;
     float readMultipleWordCostMultiplier() const;
     bool readRequiresGermanUmlautProcessing() const;
 

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp

@@ -130,6 +130,13 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
     return true;
 }
 
+/* static */ void HeaderReadWriteUtils::setCodePointVectorAttribute(
+        AttributeMap *const headerAttributes, const char *const key, const std::vector<int> value) {
+    AttributeMap::key_type keyVector;
+    insertCharactersIntoVector(key, &keyVector);
+    (*headerAttributes)[keyVector] = value;
+}
+
 /* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
         const char *const key, const bool value) {
     setIntAttribute(headerAttributes, key, value ? 1 : 0);
@@ -151,6 +158,18 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
     (*headerAttributes)[*key] = valueVector;
 }
 
+/* static */ const std::vector<int> HeaderReadWriteUtils::readCodePointVectorAttributeValue(
+        const AttributeMap *const headerAttributes, const char *const key) {
+    AttributeMap::key_type keyVector;
+    insertCharactersIntoVector(key, &keyVector);
+    AttributeMap::const_iterator it = headerAttributes->find(keyVector);
+    if (it == headerAttributes->end()) {
+        return std::vector<int>();
+    } else {
+        return it->second;
+    }
+}
+
 /* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
         const AttributeMap *const headerAttributes, const char *const key,
         const bool defaultValue) {

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h

@@ -63,12 +63,18 @@ class HeaderReadWriteUtils {
     /**
      * Methods for header attributes.
      */
+    static void setCodePointVectorAttribute(AttributeMap *const headerAttributes,
+            const char *const key, const std::vector<int> value);
+
     static void setBoolAttribute(AttributeMap *const headerAttributes,
             const char *const key, const bool value);
 
     static void setIntAttribute(AttributeMap *const headerAttributes,
             const char *const key, const int value);
 
+    static const std::vector<int> readCodePointVectorAttributeValue(
+            const AttributeMap *const headerAttributes, const char *const key);
+
     static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
             const char *const key, const bool defaultValue);
 

native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp

@@ -31,11 +31,12 @@ namespace latinime {
 const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
 
 /* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
-        const int dictVersion, const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
+        const int dictVersion, const std::vector<int> localeAsCodePointVector,
+        const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
     TimeKeeper::setCurrentTime();
     switch (dictVersion) {
         case FormatUtils::VERSION_4:
-            return createEmptyV4DictFile(filePath, attributeMap);
+            return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap);
         default:
             AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
                     filePath, dictVersion);
@@ -44,8 +45,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
 }
 
 /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
+        const std::vector<int> localeAsCodePointVector,
         const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
-    HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
+    HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
     Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
             Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
     headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,

native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h

@@ -31,6 +31,7 @@ class DictFileWritingUtils {
     static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
 
     static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
+            const std::vector<int> localeAsCodePointVector,
             const HeaderReadWriteUtils::AttributeMap *const attributeMap);
 
     static bool flushAllHeaderAndBodyToFile(const char *const filePath,
@@ -44,6 +45,7 @@ class DictFileWritingUtils {
     DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
 
     static bool createEmptyV4DictFile(const char *const filePath,
+            const std::vector<int> localeAsCodePointVector,
             const HeaderReadWriteUtils::AttributeMap *const attributeMap);
 
     static bool flushBufferToFile(const char *const filePath,

native/jni/src/utils/char_utils.cpp

@@ -1273,4 +1273,6 @@ static int compare_pair_capital(const void *a, const void *b) {
     /* U+04F0 */ 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04F6, 0x04F7,
     /* U+04F8 */ 0x042B, 0x044B, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF,
 };
+
+/* static */ const std::vector<int> CharUtils::EMPTY_STRING(1 /* size */, '\0' /* value */);
 } // namespace latinime

native/jni/src/utils/char_utils.h

@@ -18,6 +18,7 @@
 #define LATINIME_CHAR_UTILS_H
 
 #include <cctype>
+#include <vector>
 
 #include "defines.h"
 
@@ -85,7 +86,15 @@ class CharUtils {
         return spaceCount;
     }
 
+    static AK_FORCE_INLINE std::vector<int> convertShortArrayToIntVector(
+            const unsigned short *const source, const int length) {
+        std::vector<int> destination;
+        destination.insert(destination.end(), source, source + length);
+        return destination; // Copies the vector
+    }
+
     static unsigned short latin_tolower(const unsigned short c);
+    static const std::vector<int> EMPTY_STRING;
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(CharUtils);

tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java

@@ -28,6 +28,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
 import com.android.inputmethod.latin.utils.FileUtils;
+import com.android.inputmethod.latin.utils.LocaleUtils;
 
 import java.io.File;
 import java.io.IOException;
@@ -104,15 +105,14 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
         FileUtils.deleteRecursively(file);
         Map<String, String> attributeMap = new HashMap<String, String>();
         attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId);
-        attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, dictId);
         attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
                 String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
         attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
                 DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
         attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
                 DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
-        if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+        if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
-                FormatSpec.VERSION4, attributeMap)) {
+                LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
             return file;
         } else {
             throw new IOException("Empty dictionary " + file.getAbsolutePath()

tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java

@@ -69,8 +69,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
         file.delete();
         file.mkdir();
         Map<String, String> attributeMap = new HashMap<String, String>();
-        if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
+        if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
-                FormatSpec.VERSION4, attributeMap)) {
+                Locale.ENGLISH, attributeMap)) {
             return file;
         } else {
             throw new IOException("Empty dictionary " + file.getAbsolutePath()