From f2e703eb95edc98fee2cf9e271b263fe7991e102 Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Wed, 11 Dec 2013 11:45:07 +0900 Subject: [PATCH] Read options from attributes This reads the German umlaut option from the key-attribute store, and removes all native references to the flag. This also completely removes the French ligatures option which has been unused for some time. Bug: 11281748 Change-Id: I407c02640ad0f2e01585db9087633c01507b5f3f --- java/res/raw/main_de.dict | Bin 1605727 -> 1605763 bytes .../suggest/core/dictionary/digraph_utils.cpp | 12 +------- .../suggest/core/dictionary/digraph_utils.h | 2 -- .../dictionary_header_structure_policy.h | 4 --- .../dictionary/header/header_policy.cpp | 7 +++++ .../dictionary/header/header_policy.h | 24 +++++++-------- .../header/header_read_write_utils.cpp | 28 +----------------- .../header/header_read_write_utils.h | 23 +------------- 8 files changed, 21 insertions(+), 79 deletions(-) diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict index 69796bbaadd646fecd3197cdb736e0000eade482..c65698d7b719227424afe90f4485c8f38abfdb81 100644 GIT binary patch delta 136 zcmWl`yA6V10EJQTiHh%M#|qNg5%Y&&1QjG-Nx~c!5OW8xb`A${5;rl%b3RphugmT8 zz57}X>+hDDOCF^rD?*dTK~m)LG03mQIX&%7mPJYE`QF1l*nW+$MME@2OSDBtbcG{& dqAv!*6+lYiR`T;>+DQW-! delta 100 zcmV~$OA&%#7(mhQ2L)6V#AgiFMlgYEYTONsa|5_GhXG7t6Nhu2)9@RI`u)BABa0Lf sk5_+B{e&dRB$tIOrI1o8S;<;9Qp;9$(nu?N>Es|sImuZrAFkCn1GbwYKL7v# diff --git a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp index 3271c1bfb..5f9b8f3e2 100644 --- a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp @@ -28,11 +28,8 @@ const DigraphUtils::digraph_t DigraphUtils::GERMAN_UMLAUT_DIGRAPHS[] = { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS -const DigraphUtils::digraph_t DigraphUtils::FRENCH_LIGATURES_DIGRAPHS[] = - { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE - { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = - { DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_FRENCH_LIGATURES }; + { DIGRAPH_TYPE_GERMAN_UMLAUT }; /* static */ bool DigraphUtils::hasDigraphForCodePoint( const DictionaryHeaderStructurePolicy *const headerPolicy, @@ -50,9 +47,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = if (headerPolicy->requiresGermanUmlautProcessing()) { return DIGRAPH_TYPE_GERMAN_UMLAUT; } - if (headerPolicy->requiresFrenchLigatureProcessing()) { - return DIGRAPH_TYPE_FRENCH_LIGATURES; - } return DIGRAPH_TYPE_NONE; } @@ -86,10 +80,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = *digraphs = GERMAN_UMLAUT_DIGRAPHS; return NELEMS(GERMAN_UMLAUT_DIGRAPHS); } - if (digraphType == DIGRAPH_TYPE_FRENCH_LIGATURES) { - *digraphs = FRENCH_LIGATURES_DIGRAPHS; - return NELEMS(FRENCH_LIGATURES_DIGRAPHS); - } return 0; } diff --git a/native/jni/src/suggest/core/dictionary/digraph_utils.h b/native/jni/src/suggest/core/dictionary/digraph_utils.h index 6ae16e390..bec2cd6e2 100644 --- a/native/jni/src/suggest/core/dictionary/digraph_utils.h +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.h @@ -34,7 +34,6 @@ class DigraphUtils { typedef enum { DIGRAPH_TYPE_NONE, DIGRAPH_TYPE_GERMAN_UMLAUT, - DIGRAPH_TYPE_FRENCH_LIGATURES } DigraphType; typedef struct { int first; int second; int compositeGlyph; } digraph_t; @@ -55,7 +54,6 @@ class DigraphUtils { const DigraphType digraphType, const int compositeGlyphCodePoint); static const digraph_t GERMAN_UMLAUT_DIGRAPHS[]; - static const digraph_t FRENCH_LIGATURES_DIGRAPHS[]; static const DigraphType USED_DIGRAPH_TYPES[]; }; } // namespace latinime diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index b05b7c334..b76b13971 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -31,12 +31,8 @@ class DictionaryHeaderStructurePolicy { virtual int getFormatVersionNumber() const = 0; - virtual bool supportsDynamicUpdate() const = 0; - virtual bool requiresGermanUmlautProcessing() const = 0; - virtual bool requiresFrenchLigatureProcessing() const = 0; - virtual float getMultiWordCostMultiplier() const = 0; virtual int getLastDecayedTime() const = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 133ae9f47..be7a3c228 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -20,6 +20,8 @@ namespace latinime { // Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; +const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = + "REQUIRES_GERMAN_UMLAUT_PROCESSING"; // TODO: Change attribute string to "IS_DECAYING_DICT". const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; @@ -66,6 +68,11 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const { return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast(demotionRate); } +bool HeaderPolicy::readRequiresGermanUmlautProcessing() const { + return HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, + REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false); +} + bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime, const int unigramCount, const int bigramCount, const int extendedRegionSize) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index d71e0a068..5e8b5f11e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -36,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)), mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), + mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()), mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, @@ -58,6 +59,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( attributeMap)), mSize(0), mAttributeMap(*attributeMap), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), + mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()), mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, @@ -71,7 +73,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { // Temporary dummy header. HeaderPolicy() : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0), - mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false), + mAttributeMap(), mMultiWordCostMultiplier(0.0f), + mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false), mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {} @@ -109,18 +112,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mSize; } - AK_FORCE_INLINE bool supportsDynamicUpdate() const { - return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags); - } - - AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { - return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags); - } - - AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const { - return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags); - } - AK_FORCE_INLINE float getMultiWordCostMultiplier() const { return mMultiWordCostMultiplier; } @@ -129,6 +120,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mIsDecayingDict; } + AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { + return mRequiresGermanUmlautProcessing; + } + AK_FORCE_INLINE int getLastUpdatedTime() const { return mLastUpdatedTime; } @@ -164,6 +159,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { DISALLOW_COPY_AND_ASSIGN(HeaderPolicy); static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; + static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY; static const char *const IS_DECAYING_DICT_KEY; static const char *const LAST_UPDATED_TIME_KEY; static const char *const LAST_DECAYED_TIME_KEY; @@ -179,6 +175,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int mSize; HeaderReadWriteUtils::AttributeMap mAttributeMap; const float mMultiWordCostMultiplier; + const bool mRequiresGermanUmlautProcessing; const bool mIsDecayingDict; const int mLastUpdatedTime; const int mLastDecayedTime; @@ -188,6 +185,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const bool mHasHistoricalInfoOfWords; float readMultipleWordCostMultiplier() const; + bool readRequiresGermanUmlautProcessing() const; static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes( const uint8_t *const dictBuf); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index f5eee99c3..2d4547066 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -35,22 +35,6 @@ const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2; const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4; const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; -// Flags for special processing -// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or -// something very bad (like, the apocalypse) will happen. Please update both at the same time. -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2; -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; - -// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. -const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE"; -const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = - "REQUIRES_GERMAN_UMLAUT_PROCESSING"; -const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY = - "REQUIRES_FRENCH_LIGATURE_PROCESSING"; /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) { // See the format of the header in the comment in @@ -68,17 +52,7 @@ const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY /* static */ HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( const HeaderReadWriteUtils::AttributeMap *const attributeMap) { - const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap, - REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */); - const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap, - REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */); - const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap, - SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */); - DictionaryFlags dictflags = NO_FLAGS; - dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0; - dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0; - dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0; - return dictflags; + return NO_FLAGS; } /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf, diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h index 225968323..fc24bbdd5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h @@ -37,18 +37,6 @@ class HeaderReadWriteUtils { static DictionaryFlags getFlags(const uint8_t *const dictBuf); - static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) { - return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0; - } - - static AK_FORCE_INLINE bool requiresGermanUmlautProcessing(const DictionaryFlags flags) { - return (flags & GERMAN_UMLAUT_PROCESSING_FLAG) != 0; - } - - static AK_FORCE_INLINE bool requiresFrenchLigatureProcessing(const DictionaryFlags flags) { - return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0; - } - static AK_FORCE_INLINE int getHeaderOptionsPosition() { return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE + HEADER_SIZE_FIELD_SIZE; @@ -101,17 +89,8 @@ class HeaderReadWriteUtils { static const int HEADER_FLAG_SIZE; static const int HEADER_SIZE_FIELD_SIZE; + // Value for the "flags" field. It's unused at the moment. static const DictionaryFlags NO_FLAGS; - // Flags for special processing - // Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAGS) or - // something very bad (like, the apocalypse) will happen. Please update both at the same time. - static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG; - static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG; - static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG; - - static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY; - static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY; - static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY; static void setIntAttributeInner(AttributeMap *const headerAttributes, const AttributeMap::key_type *const key, const int value);