diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index 1bc322724..bc856f113 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -17,14 +17,10 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.TreeMap; /** @@ -35,34 +31,6 @@ public abstract class AbstractDictDecoder implements DictDecoder { private static final int ERROR_CANNOT_READ = 1; private static final int ERROR_WRONG_FORMAT = 2; - protected DictionaryHeader readHeader(final DictBuffer headerBuffer) - throws IOException, UnsupportedFormatException { - if (headerBuffer == null) { - openDictBuffer(); - } - - final int version = HeaderReader.readVersion(headerBuffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("Unsupported version : " + version); - } - // TODO: Remove this field. - HeaderReader.readOptionFlags(headerBuffer); - final int headerSize = HeaderReader.readHeaderSize(headerBuffer); - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final HashMap attributes = HeaderReader.readAttributes(headerBuffer, - headerSize); - - final DictionaryHeader header = new DictionaryHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes), - new FormatOptions(version, DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( - attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)))); - return header; - } - @Override @UsedForTesting public int getTerminalPosition(final String word) throws IOException, UnsupportedFormatException { @@ -83,38 +51,6 @@ public abstract class AbstractDictDecoder implements DictDecoder { BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); } - /** - * A utility class for reading a file header. - */ - protected static class HeaderReader { - protected static int readVersion(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); - } - - protected static int readOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedShort(); - } - - protected static int readHeaderSize(final DictBuffer dictBuffer) { - return dictBuffer.readInt(); - } - - protected static HashMap readAttributes(final DictBuffer dictBuffer, - final int headerSize) { - final HashMap attributes = new HashMap(); - while (dictBuffer.position() < headerSize) { - // We can avoid an infinite loop here since dictBuffer.position() is always - // increased by calling CharEncoding.readString. - final String key = CharEncoding.readString(dictBuffer); - final String value = CharEncoding.readString(dictBuffer); - attributes.put(key, value); - } - dictBuffer.position(headerSize); - return attributes; - } - } - /** * Check whether the header contains the expected information. This is a no-error method, * that will return an error code and never throw a checked exception. @@ -148,7 +84,7 @@ public abstract class AbstractDictDecoder implements DictDecoder { } @Override - public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + public PtNodeInfo readPtNode(final int ptNodePos) { return null; } @@ -165,14 +101,4 @@ public abstract class AbstractDictDecoder implements DictDecoder { public int readPtNodeCount() { return 0; } - - @Override - public boolean readAndFollowForwardLink() { - return false; - } - - @Override - public boolean hasNextPtNodeArray() { - return false; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 845299c99..b534ebeff 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -17,18 +17,12 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Map; -import java.util.TreeMap; /** * Decodes binary files for a FusionDictionary. @@ -47,8 +41,6 @@ public final class BinaryDictDecoderUtils { // This utility class is not publicly instantiable. } - private static final int MAX_JUMPS = 12; - @UsedForTesting public interface DictBuffer { public int readUnsignedByte(); @@ -296,60 +288,21 @@ public final class BinaryDictDecoderUtils { * @param dictDecoder the dict decoder. * @param headerSize the size of the header. * @param pos the position to seek. - * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ + @UsedForTesting /* package for tests */ static WeightedString getWordAtPosition(final DictDecoder dictDecoder, - final int headerSize, final int pos, final FormatOptions formatOptions) { + final int headerSize, final int pos) { final WeightedString result; final int originalPos = dictDecoder.getPosition(); dictDecoder.setPosition(pos); - - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); - } else { - result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos, - formatOptions); - } - + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos); dictDecoder.setPosition(originalPos); return result; } - @SuppressWarnings("unused") - private static WeightedString getWordAtPositionWithParentAddress(final DictDecoder dictDecoder, - final int pos, final FormatOptions options) { - int currentPos = pos; - ProbabilityInfo probabilityInfo = null; - final StringBuilder builder = new StringBuilder(); - // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH - for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { - PtNodeInfo currentInfo; - int loopCounter = 0; - do { - dictDecoder.setPosition(currentPos); - currentInfo = dictDecoder.readPtNode(currentPos, options); - if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) { - currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - if (DBG && loopCounter++ > MAX_JUMPS) { - MakedictLog.d("Too many jumps - probably a bug"); - } - } while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)); - if (probabilityInfo == null) { - probabilityInfo = currentInfo.mProbabilityInfo; - } - builder.insert(0, - new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); - if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - return new WeightedString(builder.toString(), probabilityInfo); - } - private static WeightedString getWordAtPositionWithoutParentAddress( - final DictDecoder dictDecoder, final int headerSize, final int pos, - final FormatOptions options) { + final DictDecoder dictDecoder, final int headerSize, final int pos) { dictDecoder.setPosition(headerSize); final int count = dictDecoder.readPtNodeCount(); int groupPos = dictDecoder.getPosition(); @@ -358,7 +311,7 @@ public final class BinaryDictDecoderUtils { PtNodeInfo last = null; for (int i = count - 1; i >= 0; --i) { - PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); + PtNodeInfo info = dictDecoder.readPtNode(groupPos); groupPos = info.mEndAddress; if (info.mOriginalAddress == pos) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); @@ -389,135 +342,6 @@ public final class BinaryDictDecoderUtils { return result; } - /** - * Reads a single node array from a buffer. - * - * This methods reads the file at the current position. A node array is fully expected to start - * at the current position. - * This will recursively read other node arrays into the structure, populating the reverse - * maps on the fly and using them to keep track of already read nodes. - * - * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. - * @param headerSize the size, in bytes, of the file header. - * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. - * @param reversePtNodeMap a mapping from addresses to already read PtNodes. - * @param options file format options. - * @return the read node array with all his children already read. - */ - private static PtNodeArray readNodeArray(final DictDecoder dictDecoder, - final int headerSize, final Map reverseNodeArrayMap, - final Map reversePtNodeMap, final FormatOptions options) - throws IOException { - final ArrayList nodeArrayContents = new ArrayList(); - final int nodeArrayOriginPos = dictDecoder.getPosition(); - - do { // Scan the linked-list node. - final int count = dictDecoder.readPtNodeCount(); - int groupPos = dictDecoder.getPosition(); - for (int i = count; i > 0; --i) { // Scan the array of PtNode. - PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); - if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue; - ArrayList shortcutTargets = info.mShortcutTargets; - ArrayList bigrams = null; - if (null != info.mBigrams) { - bigrams = new ArrayList(); - for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtPosition(dictDecoder, headerSize, - bigram.mAddress, options); - final int reconstructedFrequency = - BinaryDictIOUtils.reconstructBigramFrequency(word.getProbability(), - bigram.mFrequency); - bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); - } - } - if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { - PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); - if (null == children) { - final int currentPosition = dictDecoder.getPosition(); - dictDecoder.setPosition(info.mChildrenAddress); - children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, - reversePtNodeMap, options); - dictDecoder.setPosition(currentPosition); - } - nodeArrayContents.add( - new PtNode(info.mCharacters, shortcutTargets, bigrams, - info.mProbabilityInfo, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); - } else { - nodeArrayContents.add( - new PtNode(info.mCharacters, shortcutTargets, bigrams, - info.mProbabilityInfo, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); - } - groupPos = info.mEndAddress; - } - - // reach the end of the array. - if (options.supportsDynamicUpdate()) { - final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink(); - if (!hasValidForwardLink) break; - } - } while (options.supportsDynamicUpdate() && dictDecoder.hasNextPtNodeArray()); - - final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); - nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; - nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos; - reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); - return nodeArray; - } - - /** - * Helper function to get the binary format version from the header. - * @throws IOException - */ - private static int getFormatVersion(final DictBuffer dictBuffer) - throws IOException { - final int magic = dictBuffer.readInt(); - if (FormatSpec.MAGIC_NUMBER == magic) return dictBuffer.readUnsignedShort(); - return FormatSpec.NOT_A_VERSION_NUMBER; - } - - /** - * Helper function to get and validate the binary format version. - * @throws UnsupportedFormatException - * @throws IOException - */ - static int checkFormatVersion(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - final int version = getFormatVersion(dictBuffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("This file has version " + version - + ", but this implementation does not support versions above " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - return version; - } - - /** - * Reads a buffer and returns the memory representation of the dictionary. - * - * This high-level method takes a buffer and reads its contents, populating a - * FusionDictionary structure. - * - * @param dictDecoder the dict decoder. - * @return the created dictionary. - */ - @UsedForTesting - /* package */ static FusionDictionary readDictionaryBinary(final DictDecoder dictDecoder) - throws IOException, UnsupportedFormatException { - // Read header - final DictionaryHeader fileHeader = dictDecoder.readHeader(); - - Map reverseNodeArrayMapping = new TreeMap(); - Map reversePtNodeMapping = new TreeMap(); - final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mBodyOffset, - reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions); - return new FusionDictionary(root, fileHeader.mDictionaryOptions); - } - /** * Helper method to pass a file name instead of a File object to isBinaryDictionary. */ diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 9c5d1b9d7..1593dea4f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -122,18 +122,13 @@ public class BinaryDictEncoderUtils { * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. * * @param ptNode the PtNode to compute the size of. - * @param options file format options. * @return the maximum size of the PtNode. */ - private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { - int size = getNodeHeaderSize(ptNode, options); + private static int getPtNodeMaximumSize(final PtNode ptNode) { + int size = getNodeHeaderSize(ptNode); if (ptNode.isTerminal()) { - // If terminal, one byte for the frequency or four bytes for the terminal id. - if (options.mHasTerminalId) { - size += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } else { - size += FormatSpec.PTNODE_FREQUENCY_SIZE; - } + // If terminal, one byte for the frequency. + size += FormatSpec.PTNODE_FREQUENCY_SIZE; } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address size += getShortcutListSize(ptNode.mShortcutTargets); @@ -151,19 +146,14 @@ public class BinaryDictEncoderUtils { * the containing node array, and cache it it its 'mCachedSize' member. * * @param ptNodeArray the node array to compute the maximum size of. - * @param options file format options. */ - private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray, - final FormatOptions options) { + private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray) { int size = getPtNodeCountSize(ptNodeArray); for (PtNode node : ptNodeArray.mData) { - final int nodeSize = getPtNodeMaximumSize(node, options); + final int nodeSize = getPtNodeMaximumSize(node); node.mCachedSize = nodeSize; size += nodeSize; } - if (options.supportsDynamicUpdate()) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } ptNodeArray.mCachedSize = size; } @@ -171,15 +161,9 @@ public class BinaryDictEncoderUtils { * Compute the size of the header (flag + [parent address] + characters size) of a PtNode. * * @param ptNode the PtNode of which to compute the size of the header - * @param options file format options. */ - private static int getNodeHeaderSize(final PtNode ptNode, final FormatOptions options) { - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + getPtNodeCharactersSize(ptNode); - } else { - return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); - } + private static int getNodeHeaderSize(final PtNode ptNode) { + return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); } /** @@ -379,11 +363,10 @@ public class BinaryDictEncoderUtils { * * @param ptNodeArray the node array to compute the size of. * @param dict the dictionary in which the word/attributes are to be found. - * @param formatOptions file format options. * @return false if none of the cached addresses inside the node array changed, true otherwise. */ private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray, - final FusionDictionary dict, final FormatOptions formatOptions) { + final FusionDictionary dict) { boolean changed = false; int size = getPtNodeCountSize(ptNodeArray); for (PtNode ptNode : ptNodeArray.mData) { @@ -391,37 +374,26 @@ public class BinaryDictEncoderUtils { if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) { changed = true; } - int nodeSize = getNodeHeaderSize(ptNode, formatOptions); + int nodeSize = getNodeHeaderSize(ptNode); if (ptNode.isTerminal()) { - if (formatOptions.mHasTerminalId) { - nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } else { - nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; - } + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; } - if (formatOptions.supportsDynamicUpdate()) { - nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else if (null != ptNode.mChildren) { + if (null != ptNode.mChildren) { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, nodeSize + size, ptNode.mChildren)); } - if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) { - nodeSize += getShortcutListSize(ptNode.mShortcutTargets); - if (null != ptNode.mBigrams) { - for (WeightedString bigram : ptNode.mBigrams) { - final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, - nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, - FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); - nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; - } + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; } } ptNode.mCachedSize = nodeSize; size += nodeSize; } - if (formatOptions.supportsDynamicUpdate()) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } if (ptNodeArray.mCachedSize != size) { ptNodeArray.mCachedSize = size; changed = true; @@ -433,11 +405,10 @@ public class BinaryDictEncoderUtils { * Initializes the cached addresses of node arrays and their containing nodes from their size. * * @param flatNodes the list of node arrays. - * @param formatOptions file format options. * @return the byte size of the entire stack. */ - private static int initializePtNodeArraysCachedAddresses(final ArrayList flatNodes, - final FormatOptions formatOptions) { + private static int initializePtNodeArraysCachedAddresses( + final ArrayList flatNodes) { int nodeArrayOffset = 0; for (final PtNodeArray nodeArray : flatNodes) { nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; @@ -467,28 +438,6 @@ public class BinaryDictEncoderUtils { } } - /** - * Compute the cached parent addresses after all has been updated. - * - * The parent addresses are used by some binary formats at write-to-disk time. Not all formats - * need them. In particular, version 2 does not need them, and version 3 does. - * - * @param flatNodes the flat array of node arrays to fill in - */ - private static void computeParentAddresses(final ArrayList flatNodes) { - for (final PtNodeArray nodeArray : flatNodes) { - for (final PtNode ptNode : nodeArray.mData) { - if (null != ptNode.mChildren) { - // Assign my address to children's parent address - // Here BeforeUpdate and AfterUpdate addresses have the same value, so it - // does not matter which we use. - ptNode.mChildren.mCachedParentAddress = ptNode.mCachedAddressAfterUpdate - - ptNode.mChildren.mCachedAddressAfterUpdate; - } - } - } - } - /** * Compute the addresses and sizes of an ordered list of PtNode arrays. * @@ -501,14 +450,15 @@ public class BinaryDictEncoderUtils { * * @param dict the dictionary * @param flatNodes the ordered list of PtNode arrays - * @param formatOptions file format options. * @return the same array it was passed. The nodes have been updated for address and size. */ /* package */ static ArrayList computeAddresses(final FusionDictionary dict, - final ArrayList flatNodes, final FormatOptions formatOptions) { + final ArrayList flatNodes) { // First get the worst possible sizes and offsets - for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions); - final int offset = initializePtNodeArraysCachedAddresses(flatNodes, formatOptions); + for (final PtNodeArray n : flatNodes) { + calculatePtNodeArrayMaximumSize(n); + } + final int offset = initializePtNodeArraysCachedAddresses(flatNodes); MakedictLog.i("Compressing the array addresses. Original size : " + offset); MakedictLog.i("(Recursively seen size : " + offset + ")"); @@ -521,8 +471,7 @@ public class BinaryDictEncoderUtils { for (final PtNodeArray ptNodeArray : flatNodes) { ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset; final int oldNodeArraySize = ptNodeArray.mCachedSize; - final boolean changed = - computeActualPtNodeArraySize(ptNodeArray, dict, formatOptions); + final boolean changed = computeActualPtNodeArraySize(ptNodeArray, dict); final int newNodeArraySize = ptNodeArray.mCachedSize; if (oldNodeArraySize < newNodeArraySize) { throw new RuntimeException("Increased size ?!"); @@ -535,9 +484,6 @@ public class BinaryDictEncoderUtils { if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); } while (changesDone); - if (formatOptions.supportsDynamicUpdate()) { - computeParentAddresses(flatNodes); - } final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); MakedictLog.i("Compression complete in " + passes + " passes."); MakedictLog.i("After address compression : " @@ -634,35 +580,29 @@ public class BinaryDictEncoderUtils { * @param hasBigrams whether the PtNode has bigrams. * @param isNotAWord whether the PtNode is not a word. * @param isBlackListEntry whether the PtNode is a blacklist entry. - * @param formatOptions file format options. * @return the flags */ static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) { + final boolean isNotAWord, final boolean isBlackListEntry) { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.supportsDynamicUpdate()) { - flags |= FormatSpec.FLAG_IS_NOT_MOVED; - } else if (true) { - switch (childrenAddressSize) { - case 1: - flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; - break; - case 0: - flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; - break; - default: - throw new RuntimeException("Node with a strange address"); - } + switch (childrenAddressSize) { + case 1: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; + break; + case 0: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; + break; + default: + throw new RuntimeException("Node with a strange address"); } if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; @@ -671,12 +611,12 @@ public class BinaryDictEncoderUtils { return flags; } - /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset, - final FormatOptions formatOptions) { + /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) { return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(), getByteSize(childrenOffset), node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), - node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions); + node.mBigrams != null && !node.mBigrams.isEmpty(), + node.mIsNotAWord, node.mIsBlacklistEntry); } /** @@ -767,38 +707,14 @@ public class BinaryDictEncoderUtils { + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); } - /* package */ static final int writeParentAddress(final byte[] buffer, final int index, - final int address, final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - if (address == FormatSpec.NO_PARENT_ADDRESS) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - assert(absAddress <= FormatSpec.SINT24_MAX); - buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0) - | ((absAddress >> 16) & 0xFF)); - buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); - buffer[index + 2] = (byte)(absAddress & 0xFF); - } - return index + 3; - } else { - return index; - } - } - - /* package */ static final int getChildrenPosition(final PtNode ptNode, - final FormatOptions formatOptions) { + /* package */ static final int getChildrenPosition(final PtNode ptNode) { int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate - + getNodeHeaderSize(ptNode, formatOptions); + + getNodeHeaderSize(ptNode); if (ptNode.isTerminal()) { - // A terminal node has either the terminal id or the frequency. + // A terminal node has the frequency. // If positionOfChildrenPosField is incorrect, we may crash when jumping to the children // position. - if (formatOptions.mHasTerminalId) { - positionOfChildrenPosField += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } else { - positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; - } + positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; } return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS : ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField; @@ -810,12 +726,10 @@ public class BinaryDictEncoderUtils { * @param dict the dictionary the node array is a part of (for relative offsets). * @param dictEncoder the dictionary encoder. * @param ptNodeArray the node array to write. - * @param formatOptions file format options. */ @SuppressWarnings("unused") /* package */ static void writePlacedPtNodeArray(final FusionDictionary dict, - final DictEncoder dictEncoder, final PtNodeArray ptNodeArray, - final FormatOptions formatOptions) { + final DictEncoder dictEncoder, final PtNodeArray ptNodeArray) { // TODO: Make the code in common with BinaryDictIOUtils#writePtNode dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate); @@ -838,10 +752,7 @@ public class BinaryDictEncoderUtils { + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mProbabilityInfo.toString()); } - dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); - } - if (formatOptions.supportsDynamicUpdate()) { - dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); + dictEncoder.writePtNode(ptNode, dict); } if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize) { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 54446df49..989ca4b2f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -18,7 +18,6 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; @@ -60,8 +59,7 @@ public final class BinaryDictIOUtils { private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder, final int bodyOffset, final Map words, final Map frequencies, - final Map> bigrams, - final FormatOptions formatOptions) { + final Map> bigrams) { int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack stack = new Stack(); @@ -90,17 +88,12 @@ public final class BinaryDictIOUtils { stack.pop(); continue; } - final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress, formatOptions); + final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress); for (int i = 0; i < ptNodeInfo.mCharacters.length; ++i) { pushedChars[index++] = ptNodeInfo.mCharacters[i]; } p.mPosition++; - - final boolean isMovedPtNode = isMovedPtNode(ptNodeInfo.mFlags, - formatOptions); - final boolean isDeletedPtNode = isDeletedPtNode(ptNodeInfo.mFlags, - formatOptions); - if (!isMovedPtNode && !isDeletedPtNode && ptNodeInfo.isTerminal()) {// found word + if (ptNodeInfo.isTerminal()) {// found word words.put(ptNodeInfo.mOriginalAddress, new String(pushedChars, 0, index)); frequencies.put( ptNodeInfo.mOriginalAddress, ptNodeInfo.mProbabilityInfo.mProbability); @@ -110,25 +103,13 @@ public final class BinaryDictIOUtils { } if (p.mPosition == p.mNumOfPtNode) { - if (formatOptions.supportsDynamicUpdate()) { - final boolean hasValidForwardLinkAddress = - dictDecoder.readAndFollowForwardLink(); - if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { - // The node array has a forward link. - p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT; - p.mAddress = dictDecoder.getPosition(); - } else { - stack.pop(); - } - } else { - stack.pop(); - } + stack.pop(); } else { - // The Ptnode array has more PtNodes. + // The PtNode array has more PtNodes. p.mAddress = dictDecoder.getPosition(); } - if (!isMovedPtNode && hasChildrenAddress(ptNodeInfo.mChildrenAddress)) { + if (hasChildrenAddress(ptNodeInfo.mChildrenAddress)) { final Position childrenPos = new Position(ptNodeInfo.mChildrenAddress, index); stack.push(childrenPos); } @@ -153,7 +134,7 @@ public final class BinaryDictIOUtils { // Read header final DictionaryHeader header = dictDecoder.readHeader(); readUnigramsAndBigramsBinaryInner(dictDecoder, header.mBodyOffset, words, - frequencies, bigrams, header.mFormatOptions); + frequencies, bigrams); } /** @@ -171,8 +152,7 @@ public final class BinaryDictIOUtils { final String word) throws IOException, UnsupportedFormatException { if (word == null) return FormatSpec.NOT_VALID_WORD; dictDecoder.setPosition(0); - - final DictionaryHeader header = dictDecoder.readHeader(); + dictDecoder.readHeader(); int wordPos = 0; final int wordLen = word.codePointCount(0, word.length()); for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { @@ -183,13 +163,7 @@ public final class BinaryDictIOUtils { boolean foundNextPtNode = false; for (int i = 0; i < ptNodeCount; ++i) { final int ptNodePos = dictDecoder.getPosition(); - final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos, - header.mFormatOptions); - final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags, - header.mFormatOptions); - final boolean isDeletedNode = isDeletedPtNode(currentInfo.mFlags, - header.mFormatOptions); - if (isMovedNode) continue; + final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos); boolean same = true; for (int p = 0, j = word.offsetByCodePoints(0, wordPos); p < currentInfo.mCharacters.length; @@ -204,7 +178,7 @@ public final class BinaryDictIOUtils { if (same) { // found the PtNode matches the word. if (wordPos + currentInfo.mCharacters.length == wordLen) { - if (!currentInfo.isTerminal() || isDeletedNode) { + if (!currentInfo.isTerminal()) { return FormatSpec.NOT_VALID_WORD; } else { return ptNodePos; @@ -219,64 +193,13 @@ public final class BinaryDictIOUtils { break; } } - - // If we found the next PtNode, it is under the file pointer. - // But if not, we are at the end of this node array so we expect to have - // a forward link address that we need to consult and possibly resume - // search on the next node array in the linked list. if (foundNextPtNode) break; - if (!header.mFormatOptions.supportsDynamicUpdate()) { - return FormatSpec.NOT_VALID_WORD; - } - - final boolean hasValidForwardLinkAddress = - dictDecoder.readAndFollowForwardLink(); - if (!hasValidForwardLinkAddress || !dictDecoder.hasNextPtNodeArray()) { - return FormatSpec.NOT_VALID_WORD; - } + return FormatSpec.NOT_VALID_WORD; } while(true); } return FormatSpec.NOT_VALID_WORD; } - /** - * @return the size written, in bytes. Always 3 bytes. - */ - @UsedForTesting - static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { - final int absValue = Math.abs(value); - dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); - dictBuffer.put((byte)((absValue >> 8) & 0xFF)); - dictBuffer.put((byte)(absValue & 0xFF)); - return 3; - } - - /** - * @return the size written, in bytes. Always 3 bytes. - */ - @UsedForTesting - static int writeSInt24ToStream(final OutputStream destination, final int value) - throws IOException { - final int absValue = Math.abs(value); - destination.write((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); - destination.write((byte)((absValue >> 8) & 0xFF)); - destination.write((byte)(absValue & 0xFF)); - return 3; - } - - @UsedForTesting - static void skipString(final DictBuffer dictBuffer, - final boolean hasMultipleChars) { - if (hasMultipleChars) { - int character = CharEncoding.readChar(dictBuffer); - while (character != FormatSpec.INVALID_CHARACTER) { - character = CharEncoding.readChar(dictBuffer); - } - } else { - CharEncoding.readChar(dictBuffer); - } - } - /** * Writes a PtNodeCount to the stream. * @@ -355,30 +278,6 @@ public final class BinaryDictIOUtils { return FormatSpec.NO_CHILDREN_ADDRESS != address; } - /** - * Helper method to check whether the node is moved. - */ - public static boolean isMovedPtNode(final int flags, final FormatOptions options) { - return options.supportsDynamicUpdate() - && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); - } - - /** - * Helper method to check whether the dictionary can be updated dynamically. - */ - public static boolean supportsDynamicUpdate(final FormatOptions options) { - return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.supportsDynamicUpdate(); - } - - /** - * Helper method to check whether the node is deleted. - */ - public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { - return formatOptions.supportsDynamicUpdate() - && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); - } - /** * Compute the binary size of the node count * @param count the node count @@ -396,9 +295,7 @@ public final class BinaryDictIOUtils { } } - static int getChildrenAddressSize(final int optionFlags, - final FormatOptions formatOptions) { - if (formatOptions.supportsDynamicUpdate()) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + static int getChildrenAddressSize(final int optionFlags) { switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return 1; @@ -419,6 +316,7 @@ public final class BinaryDictIOUtils { * @param bigramFrequency compressed frequency * @return approximate bigram frequency */ + @UsedForTesting public static int reconstructBigramFrequency(final int unigramFrequency, final int bigramFrequency) { final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 0fb98aba9..a3b28a702 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -18,7 +18,6 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; @@ -45,10 +44,9 @@ public interface DictDecoder { /** * Reads PtNode from ptNodePos. * @param ptNodePos the position of PtNode. - * @param formatOptions the format options. * @return PtNodeInfo. */ - public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); + public PtNodeInfo readPtNode(final int ptNodePos); /** * Reads a buffer and returns the memory representation of the dictionary. @@ -112,14 +110,6 @@ public interface DictDecoder { */ public int readPtNodeCount(); - /** - * Reads the forward link and advances the position. - * - * @return true if this method moves the file pointer, false otherwise. - */ - public boolean readAndFollowForwardLink(); - public boolean hasNextPtNodeArray(); - /** * Opens the dictionary file and makes DictBuffer. */ diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java index ea5d492d8..a5dc45691 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -32,7 +32,5 @@ public interface DictEncoder { public int getPosition(); public void writePtNodeCount(final int ptNodeCount); public void writeForwardLinkAddress(final int forwardLinkAddress); - - public void writePtNode(final PtNode ptNode, final int parentPosition, - final FormatOptions formatOptions, final FusionDictionary dict); + public void writePtNode(final PtNode ptNode, final FusionDictionary dict); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 5a3807389..c7635eff9 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -309,7 +309,6 @@ public final class FormatSpec { */ public static final class FormatOptions { public final int mVersion; - public final boolean mHasTerminalId; public final boolean mHasTimestamp; @UsedForTesting @@ -319,13 +318,8 @@ public final class FormatSpec { public FormatOptions(final int version, final boolean hasTimestamp) { mVersion = version; - mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); mHasTimestamp = hasTimestamp; } - - public boolean supportsDynamicUpdate() { - return mVersion >= FIRST_VERSION_WITH_DYNAMIC_UPDATE; - } } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java index 4760aa8d5..f52117c6c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java +++ b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java @@ -31,12 +31,11 @@ public final class PtNodeInfo { public final int[] mCharacters; public final ProbabilityInfo mProbabilityInfo; public final int mChildrenAddress; - public final int mParentAddress; public final ArrayList mShortcutTargets; public final ArrayList mBigrams; public PtNodeInfo(final int originalAddress, final int endAddress, final int flags, - final int[] characters, final ProbabilityInfo probabilityInfo, final int parentAddress, + final int[] characters, final ProbabilityInfo probabilityInfo, final int childrenAddress, final ArrayList shortcutTargets, final ArrayList bigrams) { mOriginalAddress = originalAddress; @@ -44,7 +43,6 @@ public final class PtNodeInfo { mFlags = flags; mCharacters = characters; mProbabilityInfo = probabilityInfo; - mParentAddress = parentAddress; mChildrenAddress = childrenAddress; mShortcutTargets = shortcutTargets; mBigrams = bigrams; diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index a612ea104..315bd8e58 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -173,21 +172,23 @@ public class Ver2DictDecoder extends AbstractDictDecoder { @Override public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { - if (mDictBuffer == null) { + final DictionaryHeader header = mBinaryDictionary.getHeader(); + if (header.mFormatOptions.mVersion != FormatSpec.VERSION2) { + throw new UnsupportedFormatException("File header has a wrong version : " + + header.mFormatOptions.mVersion); + } + if (!isDictBufferOpen()) { openDictBuffer(); } - final DictionaryHeader header = super.readHeader(mDictBuffer); - final int version = header.mFormatOptions.mVersion; - if (version != FormatSpec.VERSION2) { - throw new UnsupportedFormatException("File header has a wrong version : " + version); - } + // Advance buffer reading position to the head of dictionary body. + setPosition(header.mBodyOffset); return header; } // TODO: Make this buffer multi thread safe. private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; @Override - public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + public PtNodeInfo readPtNode(final int ptNodePos) { int addressPointer = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; @@ -221,7 +222,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { childrenAddress += addressPointer; } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags); final ArrayList shortcutTargets; if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { // readShortcut will add shortcuts to shortcutTargets. @@ -244,7 +245,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { bigrams = null; } return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo, - FormatSpec.NO_PARENT_ADDRESS, childrenAddress, shortcutTargets, bigrams); + childrenAddress, shortcutTargets, bigrams); } @Override @@ -307,19 +308,4 @@ public class Ver2DictDecoder extends AbstractDictDecoder { public int readPtNodeCount() { return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); } - - @Override - public boolean readAndFollowForwardLink() { - final int nextAddress = mDictBuffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { - mDictBuffer.position(nextAddress); - return true; - } - return false; - } - - @Override - public boolean hasNextPtNodeArray() { - return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java index 634f63137..e247f0121 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java @@ -95,7 +95,7 @@ public class Ver2DictEncoder implements DictEncoder { ArrayList flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); MakedictLog.i("Computing addresses..."); - BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes); MakedictLog.i("Checking PtNode array..."); if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); @@ -107,7 +107,7 @@ public class Ver2DictEncoder implements DictEncoder { MakedictLog.i("Writing file..."); for (PtNodeArray nodeArray : flatNodes) { - BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray); } if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes); mOutStream.write(mBuffer, 0, mPosition); @@ -139,24 +139,13 @@ public class Ver2DictEncoder implements DictEncoder { countSize); } - private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) { - final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + private void writePtNodeFlags(final PtNode ptNode) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode); mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, - BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions), + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos), FormatSpec.PTNODE_FLAGS_SIZE); } - private void writeParentPosition(final int parentPosition, final PtNode ptNode, - final FormatOptions formatOptions) { - if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) { - mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, - parentPosition, formatOptions); - } else { - mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, - parentPosition - ptNode.mCachedAddressAfterUpdate, formatOptions); - } - } - private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition); if (hasSeveralChars) { @@ -171,15 +160,10 @@ public class Ver2DictEncoder implements DictEncoder { } } - private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { - final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.supportsDynamicUpdate()) { - mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, - childrenPos); - } else { - mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, - childrenPos); - } + private void writeChildrenPosition(final PtNode ptNode) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode); + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + childrenPos); } /** @@ -246,13 +230,11 @@ public class Ver2DictEncoder implements DictEncoder { } @Override - public void writePtNode(final PtNode ptNode, final int parentPosition, - final FormatOptions formatOptions, final FusionDictionary dict) { - writePtNodeFlags(ptNode, formatOptions); - writeParentPosition(parentPosition, ptNode, formatOptions); + public void writePtNode(final PtNode ptNode, final FusionDictionary dict) { + writePtNodeFlags(ptNode); writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); writeFrequency(ptNode.getProbability()); - writeChildrenPosition(ptNode, formatOptions); + writeChildrenPosition(ptNode); writeShortcuts(ptNode.mShortcutTargets); writeBigrams(ptNode.mBigrams, dict); } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 147844fd8..1050d1b0e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -122,7 +122,6 @@ public class Ver4DictEncoder implements DictEncoder { } @Override - public void writePtNode( - PtNode ptNode, int parentPosition, FormatOptions formatOptions, FusionDictionary dict) { + public void writePtNode(PtNode ptNode, FusionDictionary dict) { } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index a32b00280..e21e340c2 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -501,7 +501,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } if (fileHeader == null) return null; return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset, - address, fileHeader.mFormatOptions).mWord; + address).mWord; } private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word,