/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef LATINIME_BINARY_FORMAT_H #define LATINIME_BINARY_FORMAT_H #include "unigram_dictionary.h" namespace latinime { class BinaryFormat { private: const static int32_t MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; const static int32_t CHARACTER_ARRAY_TERMINATOR = 0x1F; const static int MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE = 2; public: const static int UNKNOWN_FORMAT = -1; const static int FORMAT_VERSION_1 = 1; const static uint16_t FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B1; static int detectFormat(const uint8_t* const dict); static int getGroupCountAndForwardPointer(const uint8_t* const dict, int* pos); static uint8_t getFlagsAndForwardPointer(const uint8_t* const dict, int* pos); static int32_t getCharCodeAndForwardPointer(const uint8_t* const dict, int* pos); static int readFrequencyWithoutMovingPointer(const uint8_t* const dict, const int pos); static int skipOtherCharacters(const uint8_t* const dict, const int pos); static int skipAttributes(const uint8_t* const dict, const int pos); static int skipChildrenPosition(const uint8_t flags, const int pos); static int skipFrequency(const uint8_t flags, const int pos); static int skipAllAttributes(const uint8_t* const dict, const uint8_t flags, const int pos); static int skipChildrenPosAndAttributes(const uint8_t* const dict, const uint8_t flags, const int pos); static int readChildrenPosition(const uint8_t* const dict, const uint8_t flags, const int pos); static bool hasChildrenInFlags(const uint8_t flags); static int getAttributeAddressAndForwardPointer(const uint8_t* const dict, const uint8_t flags, int *pos); }; inline int BinaryFormat::detectFormat(const uint8_t* const dict) { const uint16_t magicNumber = (dict[0] << 8) + dict[1]; // big endian if (FORMAT_VERSION_1_MAGIC_NUMBER == magicNumber) return FORMAT_VERSION_1; return UNKNOWN_FORMAT; } inline int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t* const dict, int* pos) { return dict[(*pos)++]; } inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t* const dict, int* pos) { return dict[(*pos)++]; } inline int32_t BinaryFormat::getCharCodeAndForwardPointer(const uint8_t* const dict, int* pos) { const int origin = *pos; const int32_t character = dict[origin]; if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { if (character == CHARACTER_ARRAY_TERMINATOR) { *pos = origin + 1; return NOT_A_CHARACTER; } else { *pos = origin + 3; const int32_t char_1 = character << 16; const int32_t char_2 = char_1 + (dict[origin + 1] << 8); return char_2 + dict[origin + 2]; } } else { *pos = origin + 1; return character; } } inline int BinaryFormat::readFrequencyWithoutMovingPointer(const uint8_t* const dict, const int pos) { return dict[pos]; } inline int BinaryFormat::skipOtherCharacters(const uint8_t* const dict, const int pos) { int currentPos = pos; int32_t character = dict[currentPos++]; while (CHARACTER_ARRAY_TERMINATOR != character) { if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { currentPos += MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE; } character = dict[currentPos++]; } return currentPos; } static inline int attributeAddressSize(const uint8_t flags) { static const int ATTRIBUTE_ADDRESS_SHIFT = 4; return (flags & UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT; /* Note: this is a value-dependant optimization of what may probably be more readably written this way: switch (flags * UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE) { case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1; case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2; case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3; default: return 0; } */ } inline int BinaryFormat::skipAttributes(const uint8_t* const dict, const int pos) { int currentPos = pos; uint8_t flags = getFlagsAndForwardPointer(dict, ¤tPos); while (flags & UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT) { currentPos += attributeAddressSize(flags); flags = getFlagsAndForwardPointer(dict, ¤tPos); } currentPos += attributeAddressSize(flags); return currentPos; } static inline int childrenAddressSize(const uint8_t flags) { static const int CHILDREN_ADDRESS_SHIFT = 6; return (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags) >> CHILDREN_ADDRESS_SHIFT; /* See the note in attributeAddressSize. The same applies here */ } inline int BinaryFormat::skipChildrenPosition(const uint8_t flags, const int pos) { return pos + childrenAddressSize(flags); } inline int BinaryFormat::skipFrequency(const uint8_t flags, const int pos) { return UnigramDictionary::FLAG_IS_TERMINAL & flags ? pos + 1 : pos; } inline int BinaryFormat::skipAllAttributes(const uint8_t* const dict, const uint8_t flags, const int pos) { // This function skips all attributes. The format makes provision for future extension // with other attributes (notably shortcuts) but for the time being, bigrams are the // only attributes that may be found in a character group, so we only look at bigrams // in this version. if (UnigramDictionary::FLAG_HAS_BIGRAMS & flags) { return skipAttributes(dict, pos); } else { return pos; } } inline int BinaryFormat::skipChildrenPosAndAttributes(const uint8_t* const dict, const uint8_t flags, const int pos) { int currentPos = pos; currentPos = skipChildrenPosition(flags, currentPos); currentPos = skipAllAttributes(dict, flags, currentPos); return currentPos; } inline int BinaryFormat::readChildrenPosition(const uint8_t* const dict, const uint8_t flags, const int pos) { int offset = 0; switch (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags) { case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: offset = dict[pos]; break; case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: offset = dict[pos] << 8; offset += dict[pos + 1]; break; case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: offset = dict[pos] << 16; offset += dict[pos + 1] << 8; offset += dict[pos + 2]; break; default: // If we come here, it means we asked for the children of a word with // no children. return -1; } return pos + offset; } inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) { return (UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags)); } inline int BinaryFormat::getAttributeAddressAndForwardPointer(const uint8_t* const dict, const uint8_t flags, int *pos) { int offset = 0; const int origin = *pos; switch (UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE & flags) { case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: offset = dict[origin]; *pos = origin + 1; break; case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: offset = dict[origin] << 8; offset += dict[origin + 1]; *pos = origin + 2; break; case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: offset = dict[origin] << 16; offset += dict[origin + 1] << 8; offset += dict[origin + 2]; *pos = origin + 3; break; } if (UnigramDictionary::FLAG_ATTRIBUTE_OFFSET_NEGATIVE & flags) { return origin - offset; } else { return origin + offset; } } } // namespace latinime #endif // LATINIME_BINARY_FORMAT_H