Read options from attributes

This reads the German umlaut option from the key-attribute
store, and removes all native references to the flag.
This also completely removes the French ligatures option which
has been unused for some time.

Bug: 11281748
Change-Id: I407c02640ad0f2e01585db9087633c01507b5f3f

native/jni/src/suggest/core/dictionary/digraph_utils.cpp

@@ -28,11 +28,8 @@ const DigraphUtils::digraph_t DigraphUtils::GERMAN_UMLAUT_DIGRAPHS[] =
         { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS
         { 'o', 'e', 0x00F6 },   // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS
         { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS
-const DigraphUtils::digraph_t DigraphUtils::FRENCH_LIGATURES_DIGRAPHS[] =
-        { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE
-        { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE
 const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
-        { DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_FRENCH_LIGATURES };
+        { DIGRAPH_TYPE_GERMAN_UMLAUT };
 
 /* static */ bool DigraphUtils::hasDigraphForCodePoint(
         const DictionaryHeaderStructurePolicy *const headerPolicy,
@@ -50,9 +47,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
     if (headerPolicy->requiresGermanUmlautProcessing()) {
         return DIGRAPH_TYPE_GERMAN_UMLAUT;
     }
-    if (headerPolicy->requiresFrenchLigatureProcessing()) {
-        return DIGRAPH_TYPE_FRENCH_LIGATURES;
-    }
     return DIGRAPH_TYPE_NONE;
 }
 
@@ -86,10 +80,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] =
         *digraphs = GERMAN_UMLAUT_DIGRAPHS;
         return NELEMS(GERMAN_UMLAUT_DIGRAPHS);
     }
-    if (digraphType == DIGRAPH_TYPE_FRENCH_LIGATURES) {
-        *digraphs = FRENCH_LIGATURES_DIGRAPHS;
-        return NELEMS(FRENCH_LIGATURES_DIGRAPHS);
-    }
     return 0;
 }
 

native/jni/src/suggest/core/dictionary/digraph_utils.h

@@ -34,7 +34,6 @@ class DigraphUtils {
     typedef enum {
         DIGRAPH_TYPE_NONE,
         DIGRAPH_TYPE_GERMAN_UMLAUT,
-        DIGRAPH_TYPE_FRENCH_LIGATURES
     } DigraphType;
 
     typedef struct { int first; int second; int compositeGlyph; } digraph_t;
@@ -55,7 +54,6 @@ class DigraphUtils {
             const DigraphType digraphType, const int compositeGlyphCodePoint);
 
     static const digraph_t GERMAN_UMLAUT_DIGRAPHS[];
-    static const digraph_t FRENCH_LIGATURES_DIGRAPHS[];
     static const DigraphType USED_DIGRAPH_TYPES[];
 };
 } // namespace latinime

native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h

@@ -31,12 +31,8 @@ class DictionaryHeaderStructurePolicy {
 
     virtual int getFormatVersionNumber() const = 0;
 
-    virtual bool supportsDynamicUpdate() const = 0;
-
     virtual bool requiresGermanUmlautProcessing() const = 0;
 
-    virtual bool requiresFrenchLigatureProcessing() const = 0;
-
     virtual float getMultiWordCostMultiplier() const = 0;
 
     virtual int getLastDecayedTime() const = 0;

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp

@@ -20,6 +20,8 @@ namespace latinime {
 
 // Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
 const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
+const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
+        "REQUIRES_GERMAN_UMLAUT_PROCESSING";
 // TODO: Change attribute string to "IS_DECAYING_DICT".
 const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
 const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date";
@@ -66,6 +68,11 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const {
     return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
 }
 
+bool HeaderPolicy::readRequiresGermanUmlautProcessing() const {
+    return HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
+            REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false);
+}
+
 bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite,
         const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime,
         const int unigramCount, const int bigramCount, const int extendedRegionSize) const {

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

@@ -36,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
               mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
               mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
               mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
+              mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
               mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
                       IS_DECAYING_DICT_KEY, false /* defaultValue */)),
               mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@@ -58,6 +59,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
               mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
                       attributeMap)), mSize(0), mAttributeMap(*attributeMap),
               mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
+              mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
               mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
                       IS_DECAYING_DICT_KEY, false /* defaultValue */)),
               mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
@@ -71,7 +73,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     // Temporary dummy header.
     HeaderPolicy()
             : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
-              mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false),
+              mAttributeMap(), mMultiWordCostMultiplier(0.0f),
+              mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
               mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
               mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
 
@@ -109,18 +112,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
         return mSize;
     }
 
-    AK_FORCE_INLINE bool supportsDynamicUpdate() const {
-        return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags);
-    }
-
-    AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
-        return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags);
-    }
-
-    AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const {
-        return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags);
-    }
-
     AK_FORCE_INLINE float getMultiWordCostMultiplier() const {
         return mMultiWordCostMultiplier;
     }
@@ -129,6 +120,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
         return mIsDecayingDict;
     }
 
+    AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const {
+        return mRequiresGermanUmlautProcessing;
+    }
+
     AK_FORCE_INLINE int getLastUpdatedTime() const {
         return mLastUpdatedTime;
     }
@@ -164,6 +159,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
 
     static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
+    static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
     static const char *const IS_DECAYING_DICT_KEY;
     static const char *const LAST_UPDATED_TIME_KEY;
     static const char *const LAST_DECAYED_TIME_KEY;
@@ -179,6 +175,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     const int mSize;
     HeaderReadWriteUtils::AttributeMap mAttributeMap;
     const float mMultiWordCostMultiplier;
+    const bool mRequiresGermanUmlautProcessing;
     const bool mIsDecayingDict;
     const int mLastUpdatedTime;
     const int mLastDecayedTime;
@@ -188,6 +185,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
     const bool mHasHistoricalInfoOfWords;
 
     float readMultipleWordCostMultiplier() const;
+    bool readRequiresGermanUmlautProcessing() const;
 
     static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
             const uint8_t *const dictBuf);

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp

@@ -35,22 +35,6 @@ const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
 const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
 
 const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
-// Flags for special processing
-// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
-// something very bad (like, the apocalypse) will happen. Please update both at the same time.
-const HeaderReadWriteUtils::DictionaryFlags
-        HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
-const HeaderReadWriteUtils::DictionaryFlags
-        HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
-const HeaderReadWriteUtils::DictionaryFlags
-        HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
-
-// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
-const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
-const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
-        "REQUIRES_GERMAN_UMLAUT_PROCESSING";
-const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY =
-        "REQUIRES_FRENCH_LIGATURE_PROCESSING";
 
 /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
     // See the format of the header in the comment in
@@ -68,17 +52,7 @@ const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY
 /* static */ HeaderReadWriteUtils::DictionaryFlags
         HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
                 const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
-    const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap,
+    return NO_FLAGS;
-            REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
-    const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
-            REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
-    const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
-            SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
-    DictionaryFlags dictflags = NO_FLAGS;
-    dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
-    dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
-    dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0;
-    return dictflags;
 }
 
 /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,

native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h

@@ -37,18 +37,6 @@ class HeaderReadWriteUtils {
 
     static DictionaryFlags getFlags(const uint8_t *const dictBuf);
 
-    static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) {
-        return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0;
-    }
-
-    static AK_FORCE_INLINE bool requiresGermanUmlautProcessing(const DictionaryFlags flags) {
-        return (flags & GERMAN_UMLAUT_PROCESSING_FLAG) != 0;
-    }
-
-    static AK_FORCE_INLINE bool requiresFrenchLigatureProcessing(const DictionaryFlags flags) {
-        return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0;
-    }
-
     static AK_FORCE_INLINE int getHeaderOptionsPosition() {
         return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE
                 + HEADER_SIZE_FIELD_SIZE;
@@ -101,17 +89,8 @@ class HeaderReadWriteUtils {
     static const int HEADER_FLAG_SIZE;
     static const int HEADER_SIZE_FIELD_SIZE;
 
+    // Value for the "flags" field. It's unused at the moment.
     static const DictionaryFlags NO_FLAGS;
-    // Flags for special processing
-    // Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAGS) or
-    // something very bad (like, the apocalypse) will happen. Please update both at the same time.
-    static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG;
-    static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG;
-    static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG;
-
-    static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY;
-    static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY;
-    static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY;
 
     static void setIntAttributeInner(AttributeMap *const headerAttributes,
             const AttributeMap::key_type *const key, const int value);