From 3bbb31f3f00e64cb68bd5877ae69d6dbccfeb519 Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Tue, 27 Mar 2012 12:36:19 +0900 Subject: [PATCH] Change the format of the shortcuts in the binary dict. This only includes the write part of the change. The read part is coming in a different commit. Change-Id: Iabe7af6cd134462dc19245f5400719920ed31c8f --- .../latin/makedict/BinaryDictInputOutput.java | 151 +++++++++++++----- 1 file changed, 107 insertions(+), 44 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index 42dd4df34..a0059e24c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -47,7 +47,6 @@ public class BinaryDictInputOutput { * s | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS - * | is shortcut only ? 1 bit, 1 = yes, 0 = no : FLAG_IS_SHORTCUT_ONLY * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers @@ -74,7 +73,7 @@ public class BinaryDictInputOutput { * dress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS - * | shortcut targets address list + * | shortcut string list * | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS * | bigrams address list * @@ -89,7 +88,7 @@ public class BinaryDictInputOutput { * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control * characters which should never happen anyway (and still work, but take 3 bytes). * - * bigram and shortcut address list is: + * bigram address list is: * = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE * | 1 = must take -address, 0 = must take +address @@ -107,8 +106,16 @@ public class BinaryDictInputOutput { * | read 3 bytes, add top 4 bits * | END * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address - * if (FLAG_ATTRIBUTE_HAS_NET) goto bigram_and_shortcut_address_list_is + * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is * + * shortcut string list is: + * = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. + * = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT + * | reserved = 3 bits, must be 0 + * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY + * = | string of characters at the char format described above, with the terminator + * | used to signal the end of the string. + * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags */ private static final int VERSION_1_MAGIC_NUMBER = 0x78B1; @@ -136,7 +143,6 @@ public class BinaryDictInputOutput { private static final int FLAG_IS_TERMINAL = 0x10; private static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08; private static final int FLAG_HAS_BIGRAMS = 0x04; - private static final int FLAG_IS_SHORTCUT_ONLY = 0x02; private static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; private static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; @@ -154,6 +160,7 @@ public class BinaryDictInputOutput { private static final int GROUP_MAX_ADDRESS_SIZE = 3; private static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; private static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; + private static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; private static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; private static final int INVALID_CHARACTER = -1; @@ -215,24 +222,52 @@ public class BinaryDictInputOutput { /** * Writes a char array to a byte buffer. * - * @param characters the character array to write. + * @param codePoints the code point array to write. * @param buffer the byte buffer to write to. * @param index the index in buffer to write the character array to. * @return the index after the last character. */ - private static int writeCharArray(int[] characters, byte[] buffer, int index) { - for (int character : characters) { - if (1 == getCharSize(character)) { - buffer[index++] = (byte)character; + private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { + for (int codePoint : codePoints) { + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; } else { - buffer[index++] = (byte)(0xFF & (character >> 16)); - buffer[index++] = (byte)(0xFF & (character >> 8)); - buffer[index++] = (byte)(0xFF & character); + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); } } return index; } + /** + * Writes a string with our character format to a byte buffer. + * + * This will also write the terminator byte. + * + * @param buffer the byte buffer to write to. + * @param origin the offset to write from. + * @param word the string to write. + * @return the size written, in bytes. + */ + private static int writeString(final byte[] buffer, final int origin, + final String word) { + final int length = word.length(); + int index = origin; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + buffer[index++] = GROUP_CHARACTERS_TERMINATOR; + return index - origin; + } + /** * Reads a character from the file. * @@ -293,6 +328,36 @@ public class BinaryDictInputOutput { return getGroupCountSize(node.mData.size()); } + /** + * Compute the size of a shortcut in bytes. + */ + private static int getShortcutSize(final WeightedString shortcut) { + int size = GROUP_ATTRIBUTE_FLAGS_SIZE; + final String word = shortcut.mWord; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + size += CharEncoding.getCharSize(codePoint); + } + size += GROUP_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the size of a shortcut list in bytes. + * + * This is known in advance and does not change according to position in the file + * like address lists do. + */ + private static int getShortcutListSize(final ArrayList shortcutList) { + if (null == shortcutList) return 0; + int size = GROUP_SHORTCUT_LIST_SIZE_SIZE; + for (final WeightedString shortcut : shortcutList) { + size += getShortcutSize(shortcut); + } + return size; + } + /** * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. * @@ -304,10 +369,7 @@ public class BinaryDictInputOutput { // If terminal, one byte for the frequency if (group.isTerminal()) size += GROUP_FREQUENCY_SIZE; size += GROUP_MAX_ADDRESS_SIZE; // For children address - if (null != group.mShortcutTargets) { - size += (GROUP_ATTRIBUTE_FLAGS_SIZE + GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) - * group.mShortcutTargets.size(); - } + size += getShortcutListSize(group.mShortcutTargets); if (null != group.mBigrams) { size += (GROUP_ATTRIBUTE_FLAGS_SIZE + GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) * group.mBigrams.size(); @@ -338,13 +400,6 @@ public class BinaryDictInputOutput { return NO_CHILDREN_ADDRESS != address; } - /** - * Helper method to find out if a character info is a shortcut only. - */ - private static boolean isShortcutOnly(final CharGroupInfo info) { - return 0 != (info.mFlags & FLAG_IS_SHORTCUT_ONLY); - } - /** * Compute the size, in bytes, that an address will occupy. * @@ -430,15 +485,7 @@ public class BinaryDictInputOutput { final int offset = group.mChildren.mCachedAddress - offsetBasePoint; groupSize += getByteSize(offset); } - if (null != group.mShortcutTargets) { - for (WeightedString target : group.mShortcutTargets) { - final int offsetBasePoint = groupSize + node.mCachedAddress + size - + GROUP_FLAGS_SIZE; - final int addressOfTarget = findAddressOfWord(dict, target.mWord); - final int offset = addressOfTarget - offsetBasePoint; - groupSize += getByteSize(offset) + GROUP_FLAGS_SIZE; - } - } + groupSize += getShortcutListSize(group.mShortcutTargets); if (null != group.mBigrams) { for (WeightedString bigram : group.mBigrams) { final int offsetBasePoint = groupSize + node.mCachedAddress + size @@ -555,7 +602,7 @@ public class BinaryDictInputOutput { * @param address the address to write. * @return the size in bytes the address actually took. */ - private static int writeVariableAddress(byte[] buffer, int index, int address) { + private static int writeVariableAddress(final byte[] buffer, int index, final int address) { switch (getByteSize(address)) { case 1: buffer[index++] = (byte)address; @@ -610,9 +657,6 @@ public class BinaryDictInputOutput { } flags |= FLAG_HAS_BIGRAMS; } - if (group.mIsShortcutOnly) { - flags |= FLAG_IS_SHORTCUT_ONLY; - } return flags; } @@ -645,6 +689,17 @@ public class BinaryDictInputOutput { return bigramFlags; } + /** + * Makes the flag value for a shortcut. + * + * @param more whether there are more attributes after this one. + * @param frequency the frequency of the attribute, 0..15 + * @return the flags + */ + private static final int makeShortcutFlags(final boolean more, final int frequency) { + return (more ? FLAG_ATTRIBUTE_HAS_NEXT : 0) + (frequency & FLAG_ATTRIBUTE_FREQUENCY); + } + /** * Write a node to memory. The node is expected to have its final position cached. * @@ -675,7 +730,8 @@ public class BinaryDictInputOutput { for (int i = 0; i < groupCount; ++i) { CharGroup group = node.mData.get(i); if (index != group.mCachedAddress) throw new RuntimeException("Bug: write index is not " - + "the same as the cached address of the group"); + + "the same as the cached address of the group : " + + index + " <> " + group.mCachedAddress); groupAddress += GROUP_FLAGS_SIZE + getGroupCharactersSize(group); // Sanity checks. if (group.mFrequency > MAX_TERMINAL_FREQUENCY) { @@ -700,19 +756,26 @@ public class BinaryDictInputOutput { // Write shortcuts if (null != group.mShortcutTargets) { + final int indexOfShortcutByteSize = index; + index += GROUP_SHORTCUT_LIST_SIZE_SIZE; + groupAddress += GROUP_SHORTCUT_LIST_SIZE_SIZE; final Iterator shortcutIterator = group.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = (WeightedString)shortcutIterator.next(); - final int addressOfTarget = findAddressOfWord(dict, target.mWord); ++groupAddress; - final int offset = addressOfTarget - groupAddress; - int shortcutFlags = makeAttributeFlags(shortcutIterator.hasNext(), offset, + int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), target.mFrequency); buffer[index++] = (byte)shortcutFlags; - final int shortcutShift = writeVariableAddress(buffer, index, Math.abs(offset)); + final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); index += shortcutShift; groupAddress += shortcutShift; } + final int shortcutByteSize = index - indexOfShortcutByteSize; + if (shortcutByteSize > 0xFFFF) { + throw new RuntimeException("Shortcut list too large"); + } + buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); + buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); } // Write bigrams if (null != group.mBigrams) { @@ -1112,11 +1175,11 @@ public class BinaryDictInputOutput { } nodeContents.add( new CharGroup(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, - children, isShortcutOnly(info))); + children, false)); } else { nodeContents.add( new CharGroup(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, - isShortcutOnly(info))); + false)); } groupOffset = info.mEndAddress; }