From 75de9fb7ff5662360be6076622bcdac25d01f90b Mon Sep 17 00:00:00 2001 From: Jean Chalard Date: Tue, 10 Dec 2013 23:08:29 +0900 Subject: [PATCH] Remove flags from Java side. This simplifies the code quite a bit. - GERMAN_UMLAUTS are now handled through a key-value attribute. The dictionary generator does not need to know about it any more. - FRENCH_LIGATURES are deprecated as we handle them with shortcuts now. - CONTAINS_BIGRAMS is deprecated. Bigram processing is always applied regardless of this flag. Bug: 11281748 Change-Id: I55a11ba61d3589c1584a3fa6c941374b349b7b5c --- .../inputmethod/latin/DictionaryWriter.java | 4 +- .../latin/makedict/AbstractDictDecoder.java | 9 +- .../makedict/BinaryDictDecoderUtils.java | 6 +- .../makedict/BinaryDictEncoderUtils.java | 28 ++-- .../latin/makedict/BinaryDictIOUtils.java | 12 +- .../makedict/DynamicBinaryDictIOUtils.java | 4 +- .../latin/makedict/FormatSpec.java | 112 +++++-------- .../latin/makedict/FusionDictionary.java | 147 +----------------- .../latin/makedict/Ver3DictEncoder.java | 2 +- .../latin/makedict/Ver4DictDecoder.java | 3 - .../latin/makedict/Ver4DictEncoder.java | 2 +- .../latin/makedict/Ver4DictUpdater.java | 3 - ...ecayingExpandableBinaryDictionaryBase.java | 2 - .../latin/utils/UserHistoryDictIOUtils.java | 3 +- .../latin/BinaryDictionaryDecayingTests.java | 2 - .../latin/BinaryDictionaryTests.java | 2 - .../latin/FusionDictionaryTests.java | 2 +- .../BinaryDictDecoderEncoderTests.java | 76 +++------ .../makedict/BinaryDictIOUtilsTests.java | 17 +- .../latin/makedict/BinaryDictUtils.java | 22 +-- .../latin/dicttool/CombinedInputOutput.java | 15 +- .../inputmethod/latin/dicttool/Diff.java | 12 -- .../latin/dicttool/XmlDictInputOutput.java | 13 +- .../BinaryDictOffdeviceUtilsTests.java | 3 +- .../BinaryDictEncoderFlattenTreeTests.java | 3 +- .../latin/makedict/FusionDictionaryTest.java | 3 +- 26 files changed, 115 insertions(+), 392 deletions(-) diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index f960c5343..89ef96d7f 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -18,8 +18,6 @@ package com.android.inputmethod.latin; import android.content.Context; -import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; @@ -52,7 +50,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { public void clear() { final HashMap attributes = CollectionUtils.newHashMap(); mFusionDictionary = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(attributes, false, false)); + new FusionDictionary.DictionaryOptions(attributes)); } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index bbbb8e461..f8fa68f45 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -58,12 +58,9 @@ public abstract class AbstractDictDecoder implements DictDecoder { headerSize); final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE), - 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); + new FusionDictionary.DictionaryOptions(attributes), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); return header; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 83ee7d685..7f0aa777f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -330,7 +330,7 @@ public final class BinaryDictDecoderUtils { static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { final int address = dictBuffer.readUnsignedInt24(); if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; if ((address & FormatSpec.MSB24) != 0) { @@ -540,11 +540,11 @@ public final class BinaryDictDecoderUtils { } // reach the end of the array. - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink(); if (!hasValidForwardLink) break; } - } while (options.mSupportsDynamicUpdate && dictDecoder.hasNextPtNodeArray()); + } while (options.supportsDynamicUpdate() && dictDecoder.hasNextPtNodeArray()); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index c0dad3db2..8ba0797de 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -20,7 +20,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -161,7 +160,7 @@ public class BinaryDictEncoderUtils { node.mCachedSize = nodeSize; size += nodeSize; } - if (options.mSupportsDynamicUpdate) { + if (options.supportsDynamicUpdate()) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } ptNodeArray.mCachedSize = size; @@ -398,7 +397,7 @@ public class BinaryDictEncoderUtils { nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; } } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, @@ -418,7 +417,7 @@ public class BinaryDictEncoderUtils { ptNode.mCachedSize = nodeSize; size += nodeSize; } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } if (ptNodeArray.mCachedSize != size) { @@ -534,7 +533,7 @@ public class BinaryDictEncoderUtils { if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); } while (changesDone); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { computeParentAddresses(flatNodes); } final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); @@ -643,7 +642,7 @@ public class BinaryDictEncoderUtils { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { flags |= FormatSpec.FLAG_IS_NOT_MOVED; } else if (true) { switch (childrenAddressSize) { @@ -755,16 +754,11 @@ public class BinaryDictEncoderUtils { } /** - * Makes the 2-byte value for options flags. + * Makes the 2-byte value for options flags. Unused at the moment, and always 0. */ - private static final int makeOptionsValue(final FusionDictionary dictionary, - final FormatOptions formatOptions) { - final DictionaryOptions options = dictionary.mOptions; - final boolean hasBigrams = dictionary.hasBigrams(); - return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) - + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) - + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) - + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); + private static final int makeOptionsValue(final FormatOptions formatOptions) { + // TODO: why doesn't this handle CONTAINS_TIMESTAMP_FLAG? + return 0; } /** @@ -852,7 +846,7 @@ public class BinaryDictEncoderUtils { } dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); } if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate @@ -953,7 +947,7 @@ public class BinaryDictEncoderUtils { headerBuffer.write((byte) (0xFF & version)); // Options flags - final int options = makeOptionsValue(dict, formatOptions); + final int options = makeOptionsValue(formatOptions); headerBuffer.write((byte) (0xFF & (options >> 8))); headerBuffer.write((byte) (0xFF & options)); final int headerSizeOffset = headerBuffer.size(); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 86ebf5844..640d778bb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -112,7 +112,7 @@ public final class BinaryDictIOUtils { } if (p.mPosition == p.mNumOfPtNode) { - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { final boolean hasValidForwardLinkAddress = dictDecoder.readAndFollowForwardLink(); if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { @@ -228,7 +228,7 @@ public final class BinaryDictIOUtils { // a forward link address that we need to consult and possibly resume // search on the next node array in the linked list. if (foundNextPtNode) break; - if (!header.mFormatOptions.mSupportsDynamicUpdate) { + if (!header.mFormatOptions.supportsDynamicUpdate()) { return FormatSpec.NOT_VALID_WORD; } @@ -507,7 +507,7 @@ public final class BinaryDictIOUtils { * Helper method to check whether the node is moved. */ public static boolean isMovedPtNode(final int flags, final FormatOptions options) { - return options.mSupportsDynamicUpdate + return options.supportsDynamicUpdate() && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); } @@ -516,14 +516,14 @@ public final class BinaryDictIOUtils { */ public static boolean supportsDynamicUpdate(final FormatOptions options) { return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.mSupportsDynamicUpdate; + && options.supportsDynamicUpdate(); } /** * Helper method to check whether the node is deleted. */ public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { - return formatOptions.mSupportsDynamicUpdate + return formatOptions.supportsDynamicUpdate() && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); } @@ -546,7 +546,7 @@ public final class BinaryDictIOUtils { static int getChildrenAddressSize(final int optionFlags, final FormatOptions formatOptions) { - if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + if (formatOptions.supportsDynamicUpdate()) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return 1; diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 971b4ff9f..ff03190a3 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -61,7 +61,7 @@ public final class DynamicBinaryDictIOUtils { final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); final int originalPosition = dictBuffer.position(); dictBuffer.position(ptNodeOriginAddress); - if (!formatOptions.mSupportsDynamicUpdate) { + if (!formatOptions.supportsDynamicUpdate()) { throw new RuntimeException("this file format does not support parent addresses"); } final int flags = dictBuffer.readUnsignedByte(); @@ -102,7 +102,7 @@ public final class DynamicBinaryDictIOUtils { } if (!dictUpdater.readAndFollowForwardLink()) break; if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break; - } while (formatOptions.mSupportsDynamicUpdate); + } while (formatOptions.supportsDynamicUpdate()); dictUpdater.setPosition(originalPosition); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 846aacf11..20ddba836 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -40,12 +40,8 @@ public final class FormatSpec { * p | not used 3 bits * t | each unigram and bigram entry has a time stamp? * i | 1 bit, 1 = yes, 0 = no : CONTAINS_TIMESTAMP_FLAG - * o | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG - * n | FRENCH_LIGATURE_PROCESSING_FLAG - * f | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE - * l | GERMAN_UMLAUT_PROCESSING_FLAG - * a | - * gs + * o | + * nflags * * h | * e | size of the file header, 4bytes @@ -82,45 +78,36 @@ public final class FormatSpec { * s * * f | - * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) - * r | forward link address, 3byte - * w | 1 byte = bbbbbbbb match - * a | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) - * r | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte - * d | - * linkaddress + * o | forward link address, 3byte + * r | 1 byte = bbbbbbbb match + * w | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) + * a | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte + * r | + * dlinkaddress */ /* Node (FusionDictionary.PtNode) layout is as follows: - * | IF !SUPPORTS_DYNAMIC_UPDATE - * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE - * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS - * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE - * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES - * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES - * g | ELSE - * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED - * | the new address is stored in the same place as the parent address - * | is deleted? 10 = yes : FLAG_IS_DELETED - * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS - * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL - * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS + * | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED + * f | the new address is stored in the same place as the parent address + * l | is deleted? 10 = yes : FLAG_IS_DELETED + * a | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS + * g | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL + * s | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD * | is blacklisted ? 1 bit, 1 = yes, 0 = no : FLAG_IS_BLACKLISTED * * p | - * a | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) - * r | parent address, 3byte - * e | 1 byte = bbbbbbbb match - * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * a | This address is relative to the head of the PtNode. - * d | If the node doesn't have a parent, this field is set to 0. + * a | parent address, 3byte + * r | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * n | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte + * t | This address is relative to the head of the PtNode. + * a | If the node doesn't have a parent, this field is set to 0. * d | - * ress + * dress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers @@ -134,23 +121,16 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | IF SUPPORTS_DYNAMIC_UPDATE - * h | children address, 3 bytes - * i | 1 byte = bbbbbbbb match - * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte - * r | if this node doesn't have children, this field is set to 0. - * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) - * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType - * a | // nothing - * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType - * d | children address, 1 byte - * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType - * e | children address, 2 bytes - * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType - * s | children address, 3 bytes - * | END - * | This address is relative to the position of this field. + * c | + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | This address is relative to the position of this field. + * a | + * ddress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -214,11 +194,7 @@ public final class FormatSpec { static final int MAXIMUM_SUPPORTED_VERSION = VERSION4; // These options need to be the same numeric values as the one in the native reading code. - static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; // TODO: Make the native reading code read this variable. - static final int SUPPORTS_DYNAMIC_UPDATE = 0x2; - static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; - static final int CONTAINS_BIGRAMS_FLAG = 0x8; static final int CONTAINS_TIMESTAMP_FLAG = 0x10; // TODO: Make this value adaptative to content data, store it in the header, and @@ -339,30 +315,23 @@ public final class FormatSpec { */ public static final class FormatOptions { public final int mVersion; - public final boolean mSupportsDynamicUpdate; public final boolean mHasTerminalId; public final boolean mHasTimestamp; + @UsedForTesting public FormatOptions(final int version) { - this(version, false); + this(version, false /* hasTimestamp */); } - @UsedForTesting - public FormatOptions(final int version, final boolean supportsDynamicUpdate) { - this(version, supportsDynamicUpdate, false /* hasTimestamp */); - } - - public FormatOptions(final int version, final boolean supportsDynamicUpdate, - final boolean hasTimestamp) { + public FormatOptions(final int version, final boolean hasTimestamp) { mVersion = version; - if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { - throw new RuntimeException("Dynamic updates are only supported with versions " - + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); - } - mSupportsDynamicUpdate = supportsDynamicUpdate; mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); mHasTimestamp = hasTimestamp; } + + public boolean supportsDynamicUpdate() { + return mVersion >= FIRST_VERSION_WITH_DYNAMIC_UPDATE; + } } /** @@ -374,7 +343,6 @@ public final class FormatSpec { public final FormatOptions mFormatOptions; // Note that these are corresponding definitions in native code in latinime::HeaderPolicy // and latinime::HeaderReadWriteUtils. - public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; public static final String HAS_HISTORICAL_INFO_ATTRIBUTE = "HAS_HISTORICAL_INFO"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 3bb218bea..fdf2ae7b5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -303,14 +303,9 @@ public final class FusionDictionary implements Iterable { * Options global to the dictionary. */ public static final class DictionaryOptions { - public final boolean mGermanUmlautProcessing; - public final boolean mFrenchLigatureProcessing; public final HashMap mAttributes; - public DictionaryOptions(final HashMap attributes, - final boolean germanUmlautProcessing, final boolean frenchLigatureProcessing) { + public DictionaryOptions(final HashMap attributes) { mAttributes = attributes; - mGermanUmlautProcessing = germanUmlautProcessing; - mFrenchLigatureProcessing = frenchLigatureProcessing; } @Override public String toString() { // Convenience method @@ -339,14 +334,6 @@ public final class FusionDictionary implements Iterable { } s.append("\n"); } - if (mGermanUmlautProcessing) { - s.append(indent); - s.append("Needs German umlaut processing\n"); - } - if (mFrenchLigatureProcessing) { - s.append(indent); - s.append("Needs French ligature processing\n"); - } return s.toString(); } } @@ -700,138 +687,6 @@ public final class FusionDictionary implements Iterable { return size; } - /** - * Recursively count the number of nodes in a given branch of the trie. - * - * @param nodeArray the node array to count. - * @return the number of nodes in this branch. - */ - public static int countNodeArrays(final PtNodeArray nodeArray) { - int size = 1; - for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - PtNode ptNode = nodeArray.mData.get(i); - if (null != ptNode.mChildren) - size += countNodeArrays(ptNode.mChildren); - } - return size; - } - - // Recursively find out whether there are any bigrams. - // This can be pretty expensive especially if there aren't any (we return as soon - // as we find one, so it's much cheaper if there are bigrams) - private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { - if (null == nodeArray) return false; - for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - PtNode ptNode = nodeArray.mData.get(i); - if (null != ptNode.mBigrams) return true; - if (hasBigramsInternal(ptNode.mChildren)) return true; - } - return false; - } - - /** - * Finds out whether there are any bigrams in this dictionary. - * - * @return true if there is any bigram, false otherwise. - */ - // TODO: this is expensive especially for large dictionaries without any bigram. - // The up side is, this is always accurate and correct and uses no memory. We should - // find a more efficient way of doing this, without compromising too much on memory - // and ease of use. - public boolean hasBigrams() { - return hasBigramsInternal(mRootNodeArray); - } - - // Historically, the tails of the words were going to be merged to save space. - // However, that would prevent the code to search for a specific address in log(n) - // time so this was abandoned. - // The code is still of interest as it does add some compression to any dictionary - // that has no need for attributes. Implementations that does not read attributes should be - // able to read a dictionary with merged tails. - // Also, the following code does support frequencies, as in, it will only merges - // tails that share the same frequency. Though it would result in the above loss of - // performance while searching by address, it is still technically possible to merge - // tails that contain attributes, but this code does not take that into account - it does - // not compare attributes and will merge terminals with different attributes regardless. - public void mergeTails() { - MakedictLog.i("Do not merge tails"); - return; - -// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); -// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); -// -// final HashMap> repository = -// new HashMap>(); -// mergeTailsInner(repository, root); -// -// MakedictLog.i("Number of different pseudohashes : " + repository.size()); -// int size = 0; -// for (ArrayList a : repository.values()) { -// size += a.size(); -// } -// MakedictLog.i("Number of nodes after merge : " + (1 + size)); -// MakedictLog.i("Recursively seen nodes : " + countNodes(root)); - } - - // The following methods are used by the deactivated mergeTails() -// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { -// if (null == a && null == b) return true; -// if (null == a || null == b) return false; -// if (a.data.size() != b.data.size()) return false; -// final int size = a.data.size(); -// for (int i = size - 1; i >= 0; --i) { -// PtNode aPtNode = a.data.get(i); -// PtNode bPtNode = b.data.get(i); -// if (aPtNode.frequency != bPtNode.frequency) return false; -// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; -// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; -// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; -// if (!isEqual(aPtNode.children, bPtNode.children)) return false; -// } -// return true; -// } - -// static private HashMap> mergeTailsInner( -// final HashMap> map, final PtNodeArray nodeArray) { -// final ArrayList branches = nodeArray.data; -// final int nodeSize = branches.size(); -// for (int i = 0; i < nodeSize; ++i) { -// PtNode ptNode = branches.get(i); -// if (null != ptNode.children) { -// String pseudoHash = getPseudoHash(ptNode.children); -// ArrayList similarList = map.get(pseudoHash); -// if (null == similarList) { -// similarList = new ArrayList(); -// map.put(pseudoHash, similarList); -// } -// boolean merged = false; -// for (PtNodeArray similar : similarList) { -// if (isEqual(ptNode.children, similar)) { -// ptNode.children = similar; -// merged = true; -// break; -// } -// } -// if (!merged) { -// similarList.add(ptNode.children); -// } -// mergeTailsInner(map, ptNode.children); -// } -// } -// return map; -// } - -// private static String getPseudoHash(final PtNodeArray nodeArray) { -// StringBuilder s = new StringBuilder(); -// for (PtNode ptNode : nodeArray.data) { -// s.append(ptNode.frequency); -// for (int ch : ptNode.chars) { -// s.append(Character.toChars(ch)); -// } -// } -// return s.toString(); -// } - /** * Iterator to walk through a dictionary. * diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 5da34534e..92eb861d6 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -169,7 +169,7 @@ public class Ver3DictEncoder implements DictEncoder { private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, childrenPos); } else { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 07522b54b..3be62f066 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -183,14 +183,11 @@ public class Ver4DictDecoder extends AbstractDictDecoder { * An auxiliary class for reading bigrams. */ protected static class BigramContentReader extends SparseTableContentReader { - private final boolean mHasTimestamp; - public BigramContentReader(final String name, final File baseDir, final DictionaryBufferFactory factory, final boolean hasTimestamp) { super(name + FormatSpec.BIGRAM_FILE_EXTENSION, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, getContentFilenames(name, hasTimestamp), getContentIds(hasTimestamp), factory); - mHasTimestamp = hasTimestamp; } // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 1a245b6db..8b80ebe63 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -365,7 +365,7 @@ public class Ver4DictEncoder implements DictEncoder { private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, mTriePos, childrenPos); } else { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java index 91d9cf345..c46bc36bb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -54,8 +54,6 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { } private static class BigramContentUpdater extends SparseTableContentUpdater { - private final boolean mHasTimestamp; - public BigramContentUpdater(final String name, final File baseDir, final boolean hasTimestamp) { super(name + FormatSpec.BIGRAM_FILE_EXTENSION, @@ -63,7 +61,6 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { BigramContentReader.getContentFilenames(name, hasTimestamp), BigramContentReader.getContentIds(hasTimestamp), new DictionaryBufferFromWritableByteBufferFactory()); - mHasTimestamp = hasTimestamp; } public void insertBigramEntries(final int terminalId, final int frequency, diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index ceec3e1cb..2dc9ecd7b 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -82,8 +82,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @Override protected Map getHeaderAttributeMap() { HashMap attributeMap = new HashMap(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index a2c3ed44d..db628fe18 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -95,8 +95,7 @@ public final class UserHistoryDictIOUtils { static FusionDictionary constructFusionDictionary(final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, final HashMap options) { final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(options, false, - false)); + new FusionDictionary.DictionaryOptions(options)); int profTotal = 0; for (final String word1 : bigrams.keySet()) { final HashMap word1Bigrams = bigrams.getBigrams(word1); diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index e0276513a..c7bc55d96 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -115,8 +115,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { getContext().getCacheDir()); FileUtils.deleteRecursively(file); Map attributeMap = new HashMap(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index cfc4c762a..a26c25886 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -69,8 +69,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { file.delete(); file.mkdir(); Map attributeMap = new HashMap(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, attributeMap)) { return file; diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index cadd0f8f3..e211d94b8 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -31,7 +31,7 @@ import java.util.HashMap; public class FusionDictionaryTests extends AndroidTestCase { public void testFindWordInTree() { FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap(), false, false)); + new FusionDictionary.DictionaryOptions(new HashMap())); dict.add("abc", 10, null, false /* isNotAWord */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index e3ec2eca9..aff045c2b 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -226,8 +226,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions) { String result = " : buffer type = " + ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); - result += " : version = " + formatOptions.mVersion; - return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; + return result + " : version = " + formatOptions.mVersion; } // Tests for readDictionaryBinary and writeDictionaryBinary @@ -315,17 +314,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final List results = CollectionUtils.newArrayList(); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION2); + BinaryDictUtils.VERSION2_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -336,17 +329,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final List results = CollectionUtils.newArrayList(); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION2); + BinaryDictUtils.VERSION2_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -472,17 +459,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final ArrayList results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION2); + BinaryDictUtils.VERSION2_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -493,17 +474,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final ArrayList results = CollectionUtils.newArrayList(); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION2); + BinaryDictUtils.VERSION2_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -612,29 +587,19 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testGetTerminalPosition() { final ArrayList results = CollectionUtils.newArrayList(); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION2_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + BinaryDictUtils.VERSION3_OPTIONS); runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -668,7 +633,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testDeleteWord() throws IOException, UnsupportedFormatException { - runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION3_OPTIONS); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index 308919499..cdbcc832f 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -114,7 +114,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { formatOptions); printPtNode(currentInfo); } - if (formatOptions.mSupportsDynamicUpdate) { + if (formatOptions.supportsDynamicUpdate()) { final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); Log.d(TAG, " forwardLinkAddress = " + forwardLinkAddress); } @@ -289,8 +289,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWord() { - runTestInsertWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestInsertWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestInsertWord(BinaryDictUtils.VERSION3_OPTIONS); + runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestInsertWord(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) { @@ -329,8 +330,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_OPTIONS); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } private void runTestRandomWords(final FormatOptions formatOptions) { @@ -377,7 +379,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testRandomWords() { - runTestRandomWords(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); - runTestRandomWords(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runTestRandomWords(BinaryDictUtils.VERSION3_OPTIONS); + runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); + runTestRandomWords(BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java index ad17a7118..34226dcd3 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -29,23 +29,17 @@ public class BinaryDictUtils { public static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - public static final FormatSpec.FormatOptions VERSION2 = + public static final FormatSpec.FormatOptions VERSION2_OPTIONS = new FormatSpec.FormatOptions(FormatSpec.VERSION2); - public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION3, false /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION3, true /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */); - public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = - new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* supportsDynamicUpdate */, - true /* hasTimestamp */); + public static final FormatSpec.FormatOptions VERSION3_OPTIONS = + new FormatSpec.FormatOptions(FormatSpec.VERSION3); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITHOUT_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, false /* hasTimestamp */); + public static final FormatSpec.FormatOptions VERSION4_OPTIONS_WITH_TIMESTAMP = + new FormatSpec.FormatOptions(FormatSpec.VERSION4, true /* hasTimestamp */); public static DictionaryOptions makeDictionaryOptions(final String id, final String version) { - final DictionaryOptions options = new DictionaryOptions(new HashMap(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */); + final DictionaryOptions options = new DictionaryOptions(new HashMap()); options.mAttributes.put(FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, "en_US"); options.mAttributes.put(FileHeader.DICTIONARY_ID_ATTRIBUTE, id); options.mAttributes.put(FileHeader.DICTIONARY_VERSION_ATTRIBUTE, version); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java index 4b6716936..16f82dafd 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java @@ -50,8 +50,6 @@ public class CombinedInputOutput { private static final String NOT_A_WORD_TAG = "not_a_word"; private static final String WHITELIST_TAG = "whitelist"; private static final String OPTIONS_TAG = "options"; - private static final String GERMAN_UMLAUT_PROCESSING_OPTION = "german_umlaut_processing"; - private static final String FRENCH_LIGATURE_PROCESSING_OPTION = "french_ligature_processing"; private static final String COMMENT_LINE_STARTER = "#"; /** @@ -112,13 +110,9 @@ public class CombinedInputOutput { attributes.put(keyValue[0], keyValue[1]); } - final boolean processUmlauts = - GERMAN_UMLAUT_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); - final boolean processLigatures = - FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); attributes.remove(OPTIONS_TAG); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions( - attributes, processUmlauts, processLigatures)); + final FusionDictionary dict = + new FusionDictionary(new PtNodeArray(), new DictionaryOptions(attributes)); String line; String word = null; @@ -216,11 +210,6 @@ public class CombinedInputOutput { destination.write(options.get(DICTIONARY_TAG)); options.remove(DICTIONARY_TAG); } - if (dict.mOptions.mGermanUmlautProcessing) { - destination.write("," + OPTIONS_TAG + "=" + GERMAN_UMLAUT_PROCESSING_OPTION); - } else if (dict.mOptions.mFrenchLigatureProcessing) { - destination.write("," + OPTIONS_TAG + "=" + FRENCH_LIGATURE_PROCESSING_OPTION); - } for (final String key : dict.mOptions.mAttributes.keySet()) { final String value = dict.mOptions.mAttributes.get(key); destination.write("," + key + "=" + value); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index 66fd084cd..7ac3c67a1 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -85,18 +85,6 @@ public class Diff extends Dicttool.Command { private static void diffHeaders(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; - if (dict0.mOptions.mFrenchLigatureProcessing != dict1.mOptions.mFrenchLigatureProcessing) { - System.out.println(" French ligature processing : " - + dict0.mOptions.mFrenchLigatureProcessing + " <=> " - + dict1.mOptions.mFrenchLigatureProcessing); - hasDifferences = true; - } - else if (dict0.mOptions.mGermanUmlautProcessing != dict1.mOptions.mGermanUmlautProcessing) { - System.out.println(" German umlaut processing : " - + dict0.mOptions.mGermanUmlautProcessing + " <=> " - + dict1.mOptions.mGermanUmlautProcessing); - hasDifferences = true; - } final HashMap options1 = new HashMap(dict1.mOptions.mAttributes); for (final String optionKey : dict0.mOptions.mAttributes.keySet()) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java index 4e99bf979..d226251c1 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java @@ -57,8 +57,6 @@ public class XmlDictInputOutput { private static final String NOT_A_WORD_ATTR = "not_a_word"; private static final String OPTIONS_KEY = "options"; - private static final String GERMAN_UMLAUT_PROCESSING_OPTION = "german_umlaut_processing"; - private static final String FRENCH_LIGATURE_PROCESSING_OPTION = "french_ligature_processing"; /** * SAX handler for a unigram XML file. @@ -120,12 +118,8 @@ public class XmlDictInputOutput { attributes.put(attrName, attrs.getValue(attrIndex)); } final String optionsString = attributes.get(OPTIONS_KEY); - final boolean processUmlauts = - GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString); - final boolean processLigatures = - FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString); mDictionary = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(attributes, processUmlauts, processLigatures)); + new DictionaryOptions(attributes)); } else { mState = UNKNOWN; } @@ -361,11 +355,6 @@ public class XmlDictInputOutput { // TODO: use an XMLSerializer if this gets big destination.write(" options = dict.mOptions.mAttributes; - if (dict.mOptions.mGermanUmlautProcessing) { - destination.write(" " + OPTIONS_KEY + "=\"" + GERMAN_UMLAUT_PROCESSING_OPTION + "\""); - } else if (dict.mOptions.mFrenchLigatureProcessing) { - destination.write(" " + OPTIONS_KEY + "=\"" + FRENCH_LIGATURE_PROCESSING_OPTION + "\""); - } for (final String key : dict.mOptions.mAttributes.keySet()) { final String value = dict.mOptions.mAttributes.get(key); destination.write(" " + key + "=\"" + value + "\""); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 1baeb7a47..2b0a50464 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -44,8 +44,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { public void testGetRawDictWorks() throws IOException, UnsupportedFormatException { // Create a thrice-compressed dictionary file. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap())); dict.add("foo", TEST_FREQ, null, false /* isNotAWord */); dict.add("fta", 1, null, false /* isNotAWord */); dict.add("ftb", 1, null, false /* isNotAWord */); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java index 55058238c..c6e497615 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java @@ -32,8 +32,7 @@ public class BinaryDictEncoderFlattenTreeTests extends TestCase { // that it does not contain any duplicates. public void testFlattenNodes() { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap())); dict.add("foo", 1, null, false /* isNotAWord */); dict.add("fta", 1, null, false /* isNotAWord */); dict.add("ftb", 1, null, false /* isNotAWord */); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java index 659650a05..76dadc25c 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java @@ -96,8 +96,7 @@ public class FusionDictionaryTest extends TestCase { // that it does not contain any duplicates. public void testFusion() { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new DictionaryOptions(new HashMap(), - false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); + new DictionaryOptions(new HashMap())); final long time = System.currentTimeMillis(); prepare(time); for (int i = 0; i < sWords.size(); ++i) {