am 73ddd404: am 78409cd9: Merge "Separate shortcut targets from the trie file."

* commit '73ddd4041967babdcb201da33042e15460031faf':
  Separate shortcut targets from the trie file.

java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java

@@ -23,11 +23,11 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
 import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
@@ -219,14 +219,14 @@ public final class BinaryDictDecoderUtils {
         }
 
         /**
-         * Writes a string with our character format to a ByteArrayOutputStream.
+         * Writes a string with our character format to an OutputStream.
          *
          * This will also write the terminator byte.
          *
-         * @param buffer the ByteArrayOutputStream to write to.
+         * @param buffer the OutputStream to write to.
          * @param word the string to write.
          */
-        static void writeString(final ByteArrayOutputStream buffer, final String word) {
+        static void writeString(final OutputStream buffer, final String word) throws IOException {
             final int length = word.length();
             for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
                 final int codePoint = word.codePointAt(i);

java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java

@@ -383,8 +383,8 @@ public class BinaryDictEncoderUtils {
                 nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray,
                         nodeSize + size, ptNode.mChildren));
             }
-            nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
             if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) {
+                nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
                 if (null != ptNode.mBigrams) {
                     for (WeightedString bigram : ptNode.mBigrams) {
                         final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,

java/src/com/android/inputmethod/latin/makedict/FormatSpec.java

@@ -266,15 +266,28 @@ public final class FormatSpec {
     // tat = Terminal Address Table
     static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
     static final String BIGRAM_FILE_EXTENSION = ".bigram";
+    static final String SHORTCUT_FILE_EXTENSION = ".shortcut";
     static final String LOOKUP_TABLE_FILE_SUFFIX = "_lookup";
     static final String CONTENT_TABLE_FILE_SUFFIX = "_index";
     static final int FREQUENCY_AND_FLAGS_SIZE = 2;
     static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
+
+    // With the English main dictionary as of October 2013, the size of bigram address table is
+    // is 584KB with the block size being 4.
+    // This is 91% of that of full address table.
     static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
     static final int BIGRAM_CONTENT_COUNT = 1;
     static final int BIGRAM_FREQ_CONTENT_INDEX = 0;
     static final String BIGRAM_FREQ_CONTENT_ID = "_freq";
 
+    static final int SHORTCUT_CONTENT_COUNT = 1;
+    static final int SHORTCUT_CONTENT_INDEX = 0;
+    // With the English main dictionary as of October 2013, the size of shortcut address table is
+    // 29KB with the block size being 64.
+    // This is only 4.4% of that of full address table.
+    static final int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64;
+    static final String SHORTCUT_CONTENT_ID = "_shortcut";
+
     static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
     static final int NO_PARENT_ADDRESS = 0;
     static final int NO_FORWARD_LINK_ADDRESS = 0;

java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java

@@ -23,6 +23,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
 import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
 import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
 import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
+import com.android.inputmethod.latin.utils.CollectionUtils;
 
 import android.util.Log;
 
@@ -43,6 +44,7 @@ public class Ver4DictDecoder extends DictDecoder {
     private static final int FILETYPE_FREQUENCY = 2;
     private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3;
     private static final int FILETYPE_BIGRAM_FREQ = 4;
+    private static final int FILETYPE_SHORTCUT = 5;
 
     private final File mDictDirectory;
     private final DictionaryBufferFactory mBufferFactory;
@@ -50,7 +52,9 @@ public class Ver4DictDecoder extends DictDecoder {
     private DictBuffer mFrequencyBuffer;
     private DictBuffer mTerminalAddressTableBuffer;
     private DictBuffer mBigramBuffer;
+    private DictBuffer mShortcutBuffer;
     private SparseTable mBigramAddressTable;
+    private SparseTable mShortcutAddressTable;
 
     @UsedForTesting
     /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
@@ -89,6 +93,10 @@ public class Ver4DictDecoder extends DictDecoder {
             return new File(mDictDirectory,
                     mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION
                             + FormatSpec.BIGRAM_FREQ_CONTENT_ID);
+        } else if (fileType == FILETYPE_SHORTCUT) {
+            return new File(mDictDirectory,
+                    mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION
+                            + FormatSpec.SHORTCUT_CONTENT_ID);
         } else {
             throw new RuntimeException("Unsupported kind of file : " + fileType);
         }
@@ -102,6 +110,8 @@ public class Ver4DictDecoder extends DictDecoder {
                 getFile(FILETYPE_TERMINAL_ADDRESS_TABLE));
         mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM_FREQ));
         loadBigramAddressSparseTable();
+        mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT));
+        loadShortcutAddressSparseTable();
     }
 
     @Override
@@ -136,6 +146,18 @@ public class Ver4DictDecoder extends DictDecoder {
                 FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
     }
 
+    // TODO: Let's have something like SparseTableContentsReader in this class.
+    private void loadShortcutAddressSparseTable() throws IOException {
+        final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName()
+                + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX);
+        final File contentFile = new File(mDictDirectory, mDictDirectory.getName()
+                + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX
+                + FormatSpec.SHORTCUT_CONTENT_ID);
+        mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile,
+                new File[] { contentFile }, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE);
+    }
+
+
     protected static class PtNodeReader extends DictDecoder.PtNodeReader {
         protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) {
             frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1);
@@ -147,6 +169,23 @@ public class Ver4DictDecoder extends DictDecoder {
         }
     }
 
+    private ArrayList<WeightedString> readShortcuts(final int terminalId) {
+        if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null;
+
+        final ArrayList<WeightedString> ret = CollectionUtils.newArrayList();
+        final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX,
+                terminalId);
+        mShortcutBuffer.position(posOfShortcuts);
+        while (true) {
+            final int flags = mShortcutBuffer.readUnsignedByte();
+            final String word = CharEncoding.readString(mShortcutBuffer);
+            ret.add(new WeightedString(word,
+                    flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY));
+            if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
+        }
+        return ret;
+    }
+
     // TODO: Make this buffer thread safe.
     // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH.
     private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
@@ -197,14 +236,7 @@ public class Ver4DictDecoder extends DictDecoder {
             childrenAddress += addressPointer;
         }
         addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
-        final ArrayList<WeightedString> shortcutTargets;
+        final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId);
-        if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
-            // readShortcut will add shortcuts to shortcutTargets.
-            shortcutTargets = new ArrayList<WeightedString>();
-            addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets);
-        } else {
-            shortcutTargets = null;
-        }
 
         final ArrayList<PendingAttribute> bigrams;
         if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {

java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java

@@ -49,6 +49,7 @@ public class Ver4DictEncoder implements DictEncoder {
     private File mDictDir;
     private String mBaseFilename;
     private BigramContentWriter mBigramWriter;
+    private ShortcutContentWriter mShortcutWriter;
 
     @UsedForTesting
     public Ver4DictEncoder(final File dictPlacedDir) {
@@ -152,6 +153,39 @@ public class Ver4DictEncoder implements DictEncoder {
         }
     }
 
+    private static class ShortcutContentWriter extends SparseTableContentWriter {
+        public ShortcutContentWriter(final String name, final int initialCapacity,
+                final File baseDir) {
+            super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, FormatSpec.SHORTCUT_CONTENT_COUNT,
+                    initialCapacity, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
+                    new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION },
+                    new String[] { FormatSpec.SHORTCUT_CONTENT_ID });
+        }
+
+        public void writeShortcutForOneWord(final int terminalId,
+                final Iterator<WeightedString> shortcutIterator) throws IOException {
+            write(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId,
+                    new SparseTableContentWriterInterface() {
+                        @Override
+                        public void write(final OutputStream outStream) throws IOException {
+                            writeShortcutForOneWordInternal(outStream, shortcutIterator);
+                        }
+                    });
+        }
+
+        private void writeShortcutForOneWordInternal(final OutputStream outStream,
+                final Iterator<WeightedString> shortcutIterator) throws IOException {
+            while (shortcutIterator.hasNext()) {
+                final WeightedString target = shortcutIterator.next();
+                final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
+                        shortcutIterator.hasNext(), target.mFrequency);
+                BinaryDictEncoderUtils.writeUIntToStream(outStream, shortcutFlags,
+                        FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
+                CharEncoding.writeString(outStream, target.mWord);
+            }
+        }
+    }
+
     private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions)
             throws FileNotFoundException, IOException {
         final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
@@ -225,6 +259,8 @@ public class Ver4DictEncoder implements DictEncoder {
         writeTerminalData(flatNodes, terminalCount);
         mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir);
         writeBigrams(flatNodes, dict);
+        mShortcutWriter = new ShortcutContentWriter(mBaseFilename, terminalCount, mDictDir);
+        writeShortcuts(flatNodes);
 
         final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
         final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
@@ -306,29 +342,6 @@ public class Ver4DictEncoder implements DictEncoder {
         }
     }
 
-    private void writeShortcuts(ArrayList<WeightedString> shortcuts) {
-        if (null == shortcuts || shortcuts.isEmpty()) return;
-
-        final int indexOfShortcutByteSize = mTriePos;
-        mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE;
-        final Iterator<WeightedString> shortcutIterator = shortcuts.iterator();
-        while (shortcutIterator.hasNext()) {
-            final WeightedString target = shortcutIterator.next();
-            final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
-                    shortcutIterator.hasNext(), target.mFrequency);
-            mTrieBuf[mTriePos++] = (byte)shortcutFlags;
-            final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos,
-                    target.mWord);
-            mTriePos += shortcutShift;
-        }
-        final int shortcutByteSize = mTriePos - indexOfShortcutByteSize;
-        if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) {
-            throw new RuntimeException("Shortcut list too large : " + shortcutByteSize);
-        }
-        BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize,
-                shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
-    }
-
     private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict)
             throws IOException {
         mBigramWriter.openStreams();
@@ -343,6 +356,19 @@ public class Ver4DictEncoder implements DictEncoder {
         mBigramWriter.closeStreams();
     }
 
+    private void writeShortcuts(final ArrayList<PtNodeArray> flatNodes) throws IOException {
+        mShortcutWriter.openStreams();
+        for (final PtNodeArray nodeArray : flatNodes) {
+            for (final PtNode ptNode : nodeArray.mData) {
+                if (ptNode.mShortcutTargets != null && !ptNode.mShortcutTargets.isEmpty()) {
+                    mShortcutWriter.writeShortcutForOneWord(ptNode.mTerminalId,
+                            ptNode.mShortcutTargets.iterator());
+                }
+            }
+        }
+        mShortcutWriter.closeStreams();
+    }
+
     @Override
     public void writeForwardLinkAddress(int forwardLinkAddress) {
         mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos,
@@ -359,7 +385,6 @@ public class Ver4DictEncoder implements DictEncoder {
             writeTerminalId(ptNode.mTerminalId);
         }
         writeChildrenPosition(ptNode, formatOptions);
-        writeShortcuts(ptNode.mShortcutTargets);
     }
 
     private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,