LatinIME/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java

/*
 * Copyright (C) 2012 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.inputmethod.latin.makedict;

import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;

import java.io.File;

/**
 * Dictionary File Format Specification.
 */
public final class FormatSpec {

    /*
     * File header layout is as follows:
     *
     * v |
     * e | MAGIC_NUMBER + version of the file format, 2 bytes.
     * r |
     * sion
     *
     * o |
     * p | not used                                3 bits
     * t | each unigram and bigram entry has a time stamp?
     * i |                                         1 bit, 1 = yes, 0 = no : CONTAINS_TIMESTAMP_FLAG
     * o | has bigrams ?                           1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG
     * n | FRENCH_LIGATURE_PROCESSING_FLAG
     * f | supports dynamic updates ?              1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE
     * l | GERMAN_UMLAUT_PROCESSING_FLAG
     * a |
     * gs
     *
     * h |
     * e | size of the file header, 4bytes
     * a |   including the size of the magic number, the option flags and the header size
     * d |
     * ersize
     *
     *   | attributes list
     *
     * attributes list is:
     * <key>   = | string of characters at the char format described below, with the terminator used
     *           | to signal the end of the string.
     * <value> = | string of characters at the char format described below, with the terminator used
     *           | to signal the end of the string.
     * if the size of already read < headersize, goto key.
     *
     */

    /*
     * Node array (FusionDictionary.PtNodeArray) layout is as follows:
     *
     * n |
     * o | the number of PtNodes, 1 or 2 bytes.
     * d | 1 byte = bbbbbbbb match
     * e |   case 1xxxxxxx => xxxxxxx << 8 + next byte
     * c |   otherwise => bbbbbbbb
     * o |
     * unt
     *
     * n |
     * o | sequence of PtNodes,
     * d | the layout of each PtNode is described below.
     * e |
     * s
     *
     * f |
     * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header)
     * r |     forward link address, 3byte
     * w | 1 byte = bbbbbbbb match
     * a |   case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte)
     * r |   otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte
     * d |
     * linkaddress
     */

    /* Node (FusionDictionary.PtNode) layout is as follows:
     *   | IF !SUPPORTS_DYNAMIC_UPDATE
     *   |   addressType                    xx               : mask with MASK_CHILDREN_ADDRESS_TYPE
     *   |                          2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS
     * f |                                  01 = 1 byte      : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE
     * l |                                  10 = 2 bytes     : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES
     * a |                                  11 = 3 bytes     : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES
     * g | ELSE
     * s |   is moved ?             2 bits, 11 = no          : FLAG_IS_NOT_MOVED
     *   |                            This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES
     *   |                                  01 = yes         : FLAG_IS_MOVED
     *   |                        the new address is stored in the same place as the parent address
     *   |   is deleted?                    10 = yes         : FLAG_IS_DELETED
     *   | has several chars ?         1 bit, 1 = yes, 0 = no   : FLAG_HAS_MULTIPLE_CHARS
     *   | has a terminal ?            1 bit, 1 = yes, 0 = no   : FLAG_IS_TERMINAL
     *   | has shortcut targets ?      1 bit, 1 = yes, 0 = no   : FLAG_HAS_SHORTCUT_TARGETS
     *   | has bigrams ?               1 bit, 1 = yes, 0 = no   : FLAG_HAS_BIGRAMS
     *   | is not a word ?             1 bit, 1 = yes, 0 = no   : FLAG_IS_NOT_A_WORD
     *   | is blacklisted ?            1 bit, 1 = yes, 0 = no   : FLAG_IS_BLACKLISTED
     *
     * p |
     * a | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header)
     * r |     parent address, 3byte
     * e | 1 byte = bbbbbbbb match
     * n |   case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte)
     * t |   otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte
     * a | This address is relative to the head of the PtNode.
     * d | If the node doesn't have a parent, this field is set to 0.
     * d |
     * ress
     *
     * c | IF FLAG_HAS_MULTIPLE_CHARS
     * h |   char, char, char, char    n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers
     * a |   end                       1 byte, = 0
     * r | ELSE
     * s |   char                      1 or 3 bytes
     *   | END
     *
     * f |
     * r | IF FLAG_IS_TERMINAL
     * e |   frequency                 1 byte
     * q |
     *
     * c | IF SUPPORTS_DYNAMIC_UPDATE
     * h |   children address, 3 bytes
     * i |   1 byte = bbbbbbbb match
     * l |     case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte)
     * d |     otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte
     * r |   if this node doesn't have children, this field is set to 0.
     * e |     (see BinaryDictEncoderUtils#writeVariableSignedAddress)
     * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType
     * a |   // nothing
     * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType
     * d |   children address, 1 byte
     * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType
     * e |   children address, 2 bytes
     * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType
     * s |   children address, 3 bytes
     *   | END
     *   | This address is relative to the position of this field.
     *
     *   | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS
     *   | shortcut string list
     *   | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS
     *   | bigrams address list
     *
     * Char format is:
     * 1 byte = bbbbbbbb match
     * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
     * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
     *       unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
     *       00011111 would be outside unicode.
     * else: iso-latin-1 code
     * This allows for the whole unicode range to be encoded, including chars outside of
     * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
     * characters which should never happen anyway (and still work, but take 3 bytes).
     *
     * bigram address list is:
     * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no     : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT
     *           | addressSign = 1 bit,                 : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE
     *           |                      1 = must take -address, 0 = must take +address
     *           |                         xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE
     *           | addressFormat = 2 bits, 00 = unused  : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE
     *           |                         01 = 1 byte  : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE
     *           |                         10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES
     *           |                         11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES
     *           | 4 bits : frequency         : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY
     * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat)
     *           |   read 1 byte, add top 4 bits
     *           | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat)
     *           |   read 2 bytes, add top 4 bits
     *           | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat
     *           |   read 3 bytes, add top 4 bits
     *           | END
     *           | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address
     * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is
     *
     * shortcut string list is:
     * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes.
     * <flags>     = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT
     *               | reserved = 3 bits, must be 0
     *               | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY
     * <shortcut>  = | string of characters at the char format described above, with the terminator
     *               | used to signal the end of the string.
     * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags
     */

    public static final int MAGIC_NUMBER = 0x9BC13AFE;
    static final int MINIMUM_SUPPORTED_VERSION = 2;
    static final int MAXIMUM_SUPPORTED_VERSION = 4;
    static final int NOT_A_VERSION_NUMBER = -1;
    static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3;
    static final int FIRST_VERSION_WITH_TERMINAL_ID = 4;
    static final int VERSION3 = 3;
    static final int VERSION4 = 4;

    // These options need to be the same numeric values as the one in the native reading code.
    static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
    // TODO: Make the native reading code read this variable.
    static final int SUPPORTS_DYNAMIC_UPDATE = 0x2;
    static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
    static final int CONTAINS_BIGRAMS_FLAG = 0x8;
    static final int CONTAINS_TIMESTAMP_FLAG = 0x10;

    // TODO: Make this value adaptative to content data, store it in the header, and
    // use it in the reading code.
    static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;

    static final int PARENT_ADDRESS_SIZE = 3;
    static final int FORWARD_LINK_ADDRESS_SIZE = 3;

    // These flags are used only in the static dictionary.
    static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0;
    static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00;
    static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40;
    static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80;
    static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0;

    static final int FLAG_HAS_MULTIPLE_CHARS = 0x20;

    static final int FLAG_IS_TERMINAL = 0x10;
    static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08;
    static final int FLAG_HAS_BIGRAMS = 0x04;
    static final int FLAG_IS_NOT_A_WORD = 0x02;
    static final int FLAG_IS_BLACKLISTED = 0x01;

    // These flags are used only in the dynamic dictionary.
    static final int MASK_MOVE_AND_DELETE_FLAG = 0xC0;
    static final int FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE = 0x40;
    static final int FLAG_IS_MOVED = 0x00 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE;
    static final int FLAG_IS_NOT_MOVED = 0x80 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE;
    static final int FLAG_IS_DELETED = 0x80;

    static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80;
    static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40;
    static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30;
    static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10;
    static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20;
    static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30;
    static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F;

    static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F;

    static final int PTNODE_TERMINATOR_SIZE = 1;
    static final int PTNODE_FLAGS_SIZE = 1;
    static final int PTNODE_FREQUENCY_SIZE = 1;
    static final int PTNODE_TERMINAL_ID_SIZE = 4;
    static final int PTNODE_MAX_ADDRESS_SIZE = 3;
    static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1;
    static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3;
    static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2;

    // These values are used only by version 4 or later.
    static final String TRIE_FILE_EXTENSION = ".trie";
    static final String FREQ_FILE_EXTENSION = ".freq";
    static final String UNIGRAM_TIMESTAMP_FILE_EXTENSION = ".timestamp";
    // tat = Terminal Address Table
    static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
    static final String BIGRAM_FILE_EXTENSION = ".bigram";
    static final String SHORTCUT_FILE_EXTENSION = ".shortcut";
    static final String LOOKUP_TABLE_FILE_SUFFIX = "_lookup";
    static final String CONTENT_TABLE_FILE_SUFFIX = "_index";
    static final int FREQUENCY_AND_FLAGS_SIZE = 2;
    static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
    static final int UNIGRAM_TIMESTAMP_SIZE = 4;

    // With the English main dictionary as of October 2013, the size of bigram address table is
    // is 584KB with the block size being 4.
    // This is 91% of that of full address table.
    static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
    static final int BIGRAM_CONTENT_COUNT = 2;
    static final int BIGRAM_FREQ_CONTENT_INDEX = 0;
    static final int BIGRAM_TIMESTAMP_CONTENT_INDEX = 1;
    static final String BIGRAM_FREQ_CONTENT_ID = "_freq";
    static final String BIGRAM_TIMESTAMP_CONTENT_ID = "_timestamp";
    static final int BIGRAM_TIMESTAMP_SIZE = 4;
    static final int BIGRAM_COUNTER_SIZE = 1;
    static final int BIGRAM_LEVEL_SIZE = 1;

    static final int SHORTCUT_CONTENT_COUNT = 1;
    static final int SHORTCUT_CONTENT_INDEX = 0;
    // With the English main dictionary as of October 2013, the size of shortcut address table is
    // 29KB with the block size being 64.
    // This is only 4.4% of that of full address table.
    static final int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64;
    static final String SHORTCUT_CONTENT_ID = "_shortcut";

    static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
    static final int NO_PARENT_ADDRESS = 0;
    static final int NO_FORWARD_LINK_ADDRESS = 0;
    static final int INVALID_CHARACTER = -1;

    static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127
    // Large PtNode array size field size is 2 bytes.
    static final int LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000;
    static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767
    static final int MAX_BIGRAMS_IN_A_PTNODE = 10000;
    static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF;

    static final int MAX_TERMINAL_FREQUENCY = 255;
    static final int MAX_BIGRAM_FREQUENCY = 15;

    public static final int SHORTCUT_WHITELIST_FREQUENCY = 15;

    // This option needs to be the same numeric value as the one in binary_format.h.
    static final int NOT_VALID_WORD = -99;
    static final int SIGNED_CHILDREN_ADDRESS_SIZE = 3;

    static final int UINT8_MAX = 0xFF;
    static final int UINT16_MAX = 0xFFFF;
    static final int UINT24_MAX = 0xFFFFFF;
    static final int SINT24_MAX = 0x7FFFFF;
    static final int MSB8 = 0x80;
    static final int MSB24 = 0x800000;

    /**
     * Options about file format.
     */
    public static final class FormatOptions {
        public final int mVersion;
        public final boolean mSupportsDynamicUpdate;
        public final boolean mHasTerminalId;
        public final boolean mHasTimestamp;
        @UsedForTesting
        public FormatOptions(final int version) {
            this(version, false);
        }

        @UsedForTesting
        public FormatOptions(final int version, final boolean supportsDynamicUpdate) {
            this(version, supportsDynamicUpdate, false /* hasTimestamp */);
        }

        public FormatOptions(final int version, final boolean supportsDynamicUpdate,
                final boolean hasTimestamp) {
            mVersion = version;
            if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) {
                throw new RuntimeException("Dynamic updates are only supported with versions "
                        + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior.");
            }
            mSupportsDynamicUpdate = supportsDynamicUpdate;
            mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID);
            mHasTimestamp = hasTimestamp;
        }
    }

    /**
     * Class representing file header.
     */
    public static final class FileHeader {
        public final int mHeaderSize;
        public final DictionaryOptions mDictionaryOptions;
        public final FormatOptions mFormatOptions;
        // Note that these are corresponding definitions in native code in latinime::HeaderPolicy
        // and latinime::HeaderReadWriteUtils.
        public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";
        public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";
        public static final String ATTRIBUTE_VALUE_TRUE = "1";

        public static final String DICTIONARY_VERSION_ATTRIBUTE = "version";
        public static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";
        public static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";
        private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description";
        public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions,
                final FormatOptions formatOptions) {
            mHeaderSize = headerSize;
            mDictionaryOptions = dictionaryOptions;
            mFormatOptions = formatOptions;
        }

        // Helper method to get the locale as a String
        public String getLocaleString() {
            return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_LOCALE_ATTRIBUTE);
        }

        // Helper method to get the version String
        public String getVersion() {
            return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_VERSION_ATTRIBUTE);
        }

        // Helper method to get the dictionary ID as a String
        public String getId() {
            return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_ID_ATTRIBUTE);
        }

        // Helper method to get the description
        public String getDescription() {
            // TODO: Right now each dictionary file comes with a description in its own language.
            // It will display as is no matter the device's locale. It should be internationalized.
            return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_DESCRIPTION_ATTRIBUTE);
        }
    }

    /**
     * Returns new dictionary decoder.
     *
     * @param dictFile the dictionary file.
     * @param bufferType The type of buffer, as one of USE_* in DictDecoder.
     * @return new dictionary decoder if the dictionary file exists, otherwise null.
     */
    public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
        if (dictFile.isDirectory()) {
            return new Ver4DictDecoder(dictFile, bufferType);
        } else if (dictFile.isFile()) {
            return new Ver3DictDecoder(dictFile, bufferType);
        }
        return null;
    }

    public static DictDecoder getDictDecoder(final File dictFile,
            final DictionaryBufferFactory factory) {
        if (dictFile.isDirectory()) {
            return new Ver4DictDecoder(dictFile, factory);
        } else if (dictFile.isFile()) {
            return new Ver3DictDecoder(dictFile, factory);
        }
        return null;
    }

    public static DictDecoder getDictDecoder(final File dictFile) {
        return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
    }

    private FormatSpec() {
        // This utility class is not publicly instantiable.
    }
}