Make SparseTable have multiple content tables.

Bug: 10920165
Change-Id: Ie9008452ee292fb0b1fec66e2ffed228c4af6c3e

java/src/com/android/inputmethod/latin/makedict/SparseTable.java

@@ -17,6 +17,7 @@
 package com.android.inputmethod.latin.makedict;
 
 import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.utils.CollectionUtils;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -37,35 +38,39 @@ public class SparseTable {
     /**
      * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize
      * terminals.
-     * It contains at index i = j / mBlockSize the index in mContentsTable where the values for
+     * It contains at index i = j / mBlockSize the index in each ArrayList in mContentsTables where
-     * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array.
+     * the values for terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized
+     * integer array.
      */
     private final ArrayList<Integer> mLookupTable;
-    private final ArrayList<Integer> mContentTable;
+    private final ArrayList<ArrayList<Integer>> mContentTables;
 
     private final int mBlockSize;
+    private final int mContentTableCount;
     public static final int NOT_EXIST = -1;
+    public static final int SIZE_OF_INT_IN_BYTES = 4;
 
     @UsedForTesting
-    public SparseTable(final int initialCapacity, final int blockSize) {
+    public SparseTable(final int initialCapacity, final int blockSize,
+            final int contentTableCount) {
         mBlockSize = blockSize;
         final int lookupTableSize = initialCapacity / mBlockSize
                 + (initialCapacity % mBlockSize > 0 ? 1 : 0);
         mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST));
-        mContentTable = new ArrayList<Integer>();
+        mContentTableCount = contentTableCount;
+        mContentTables = CollectionUtils.newArrayList();
+        for (int i = 0; i < mContentTableCount; ++i) {
+            mContentTables.add(new ArrayList<Integer>());
+        }
     }
 
     @UsedForTesting
-    public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) {
+    public SparseTable(final ArrayList<Integer> lookupTable,
+            final ArrayList<ArrayList<Integer>> contentTables, final int blockSize) {
         mBlockSize = blockSize;
-        mLookupTable = new ArrayList<Integer>(lookupTable.length);
+        mContentTableCount = contentTables.size();
-        for (int i = 0; i < lookupTable.length; ++i) {
+        mLookupTable = lookupTable;
-            mLookupTable.add(lookupTable[i]);
+        mContentTables = contentTables;
-        }
-        mContentTable = new ArrayList<Integer>(contentTable.length);
-        for (int i = 0; i < contentTable.length; ++i) {
-            mContentTable.add(contentTable[i]);
-        }
     }
 
     /**
@@ -75,8 +80,8 @@ public class SparseTable {
      * Otherwise, IndexOutOfBoundsException will be raised.
      */
     @UsedForTesting
-    private static void convertByteArrayToIntegerArray(final byte[] byteArray,
+    private static ArrayList<Integer> convertByteArrayToIntegerArray(final byte[] byteArray) {
-            final ArrayList<Integer> integerArray) {
+        final ArrayList<Integer> integerArray = new ArrayList<Integer>(byteArray.length / 4);
         for (int i = 0; i < byteArray.length; i += 4) {
             int value = 0;
             for (int j = i; j < i + 4; ++j) {
@@ -85,39 +90,43 @@ public class SparseTable {
              }
             integerArray.add(value);
         }
+        return integerArray;
     }
 
     @UsedForTesting
-    public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) {
+    public int get(final int contentTableIndex, final int index) {
-        mBlockSize = blockSize;
+        if (!contains(index)) {
-        mLookupTable = new ArrayList<Integer>(lookupTable.length / 4);
-        mContentTable = new ArrayList<Integer>(contentTable.length / 4);
-        convertByteArrayToIntegerArray(lookupTable, mLookupTable);
-        convertByteArrayToIntegerArray(contentTable, mContentTable);
-    }
-
-    @UsedForTesting
-    public int get(final int index) {
-        if (index < 0 || index / mBlockSize >= mLookupTable.size()
-                || mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
             return NOT_EXIST;
         }
-        return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
+        return mContentTables.get(contentTableIndex).get(
+                mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
     }
 
     @UsedForTesting
-    public void set(final int index, final int value) {
+    public ArrayList<Integer> getAll(final int index) {
-        if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+        final ArrayList<Integer> ret = CollectionUtils.newArrayList();
-            mLookupTable.set(index / mBlockSize, mContentTable.size());
+        for (int i = 0; i < mContentTableCount; ++i) {
-            for (int i = 0; i < mBlockSize; ++i) {
+            ret.add(get(i, index));
-                mContentTable.add(NOT_EXIST);
         }
-        }
+        return ret;
-        mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
     }
 
-    public void remove(final int index) {
+    @UsedForTesting
-        set(index, NOT_EXIST);
+    public void set(final int contentTableIndex, final int index, final int value) {
+        if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+            mLookupTable.set(index / mBlockSize, mContentTables.get(contentTableIndex).size());
+            for (int i = 0; i < mContentTableCount; ++i) {
+                for (int j = 0; j < mBlockSize; ++j) {
+                    mContentTables.get(i).add(NOT_EXIST);
+                }
+            }
+        }
+        mContentTables.get(contentTableIndex).set(
+                mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
+    }
+
+    public void remove(final int indexOfContent, final int index) {
+        set(indexOfContent, index, NOT_EXIST);
     }
 
     @UsedForTesting
@@ -127,7 +136,8 @@ public class SparseTable {
 
     @UsedForTesting
     /* package */ int getContentTableSize() {
-        return mContentTable.size();
+        // This class always has at least one content table.
+        return mContentTables.get(0).size();
     }
 
     @UsedForTesting
@@ -136,36 +146,51 @@ public class SparseTable {
     }
 
     public boolean contains(final int index) {
-        return get(index) != NOT_EXIST;
+        if (index < 0 || index / mBlockSize >= mLookupTable.size()
+                || mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
+            return false;
+        }
+        return true;
     }
 
     @UsedForTesting
-    public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream)
+    public void write(final OutputStream lookupOutStream, final OutputStream[] contentOutStreams)
             throws IOException {
+         if (contentOutStreams.length != mContentTableCount) {
+             throw new RuntimeException(contentOutStreams.length + " streams are given, but the"
+                     + " table has " + mContentTableCount + " content tables.");
+         }
         for (final int index : mLookupTable) {
-          BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4);
+          BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, SIZE_OF_INT_IN_BYTES);
         }
 
-        for (final int index : mContentTable) {
+        for (int i = 0; i < contentOutStreams.length; ++i) {
-            BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
+            for (final int data : mContentTables.get(i)) {
+                BinaryDictEncoderUtils.writeUIntToStream(contentOutStreams[i], data,
+                        SIZE_OF_INT_IN_BYTES);
+            }
         }
     }
 
     @UsedForTesting
-    public void writeToFiles(final File lookupTableFile, final File contentFile)
+    public void writeToFiles(final File lookupTableFile, final File[] contentFiles)
             throws IOException {
         FileOutputStream lookupTableOutStream = null;
-      FileOutputStream contentOutStream = null;
+        final FileOutputStream[] contentTableOutStreams = new FileOutputStream[mContentTableCount];
         try {
             lookupTableOutStream = new FileOutputStream(lookupTableFile);
-            contentOutStream = new FileOutputStream(contentFile);
+            for (int i = 0; i < contentFiles.length; ++i) {
-            write(lookupTableOutStream, contentOutStream);
+                contentTableOutStreams[i] = new FileOutputStream(contentFiles[i]);
+            }
+            write(lookupTableOutStream, contentTableOutStreams);
         } finally {
             if (lookupTableOutStream != null) {
                 lookupTableOutStream.close();
             }
-            if (contentOutStream != null) {
+            for (int i = 0; i < contentTableOutStreams.length; ++i) {
-                contentOutStream.close();
+                if (contentTableOutStreams[i] != null) {
+                    contentTableOutStreams[i].close();
+                }
             }
         }
     }
@@ -185,10 +210,15 @@ public class SparseTable {
     }
 
     @UsedForTesting
-    public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile,
+    public static SparseTable readFromFiles(final File lookupTableFile, final File[] contentFiles,
             final int blockSize) throws IOException {
         final byte[] lookupTable = readFileToByteArray(lookupTableFile);
-        final byte[] content = readFileToByteArray(contentFile);
+        final ArrayList<ArrayList<Integer>> contentTables =
-        return new SparseTable(lookupTable, content, blockSize);
+                new ArrayList<ArrayList<Integer>>(contentFiles.length);
+        for (int i = 0; i < contentFiles.length; ++i) {
+            contentTables.add(convertByteArrayToIntegerArray(readFileToByteArray(contentFiles[i])));
+        }
+        return new SparseTable(convertByteArrayToIntegerArray(readFileToByteArray(lookupTableFile)),
+                contentTables, blockSize);
     }
 }

java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java

@@ -131,7 +131,7 @@ public class Ver4DictDecoder extends DictDecoder {
                 mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
         final File contentFile = new File(mDictDirectory,
                 mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
-        mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile,
+        mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { contentFile },
                 FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
     }
 
@@ -208,7 +208,7 @@ public class Ver4DictDecoder extends DictDecoder {
         final ArrayList<PendingAttribute> bigrams;
         if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
             bigrams = new ArrayList<PendingAttribute>();
-            final int posOfBigrams = mBigramAddressTable.get(terminalId);
+            final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId);
             mBigramBuffer.position(posOfBigrams);
             while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
                 // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,

java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java

@@ -136,7 +136,7 @@ public class Ver4DictEncoder implements DictEncoder {
 
         writeTerminalData(flatNodes, terminalCount);
         mBigramAddressTable = new SparseTable(terminalCount,
-                FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE);
+                FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, 1 /* contentTableCount */);
         writeBigrams(flatNodes, dict);
         writeBigramAddressSparseTable();
 
@@ -231,8 +231,7 @@ public class Ver4DictEncoder implements DictEncoder {
         while (shortcutIterator.hasNext()) {
             final WeightedString target = shortcutIterator.next();
             final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
-                    shortcutIterator.hasNext(),
+                    shortcutIterator.hasNext(), target.mFrequency);
-                    target.mFrequency);
             mTrieBuf[mTriePos++] = (byte)shortcutFlags;
             final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos,
                     target.mWord);
@@ -254,7 +253,8 @@ public class Ver4DictEncoder implements DictEncoder {
             for (final PtNode ptNode : nodeArray.mData) {
                 if (ptNode.mBigrams != null) {
                     final int startPos = bigramBuffer.size();
-                    mBigramAddressTable.set(ptNode.mTerminalId, startPos);
+                    mBigramAddressTable.set(0 /* contentTableIndex */, ptNode.mTerminalId,
+                            startPos);
                     final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator();
                     while (bigramIterator.hasNext()) {
                         final WeightedString bigram = bigramIterator.next();
@@ -280,7 +280,7 @@ public class Ver4DictEncoder implements DictEncoder {
                 new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION);
         final File contentFile =
                 new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION);
-        mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile);
+        mBigramAddressTable.writeToFiles(lookupIndexFile, new File[] { contentFile });
     }
 
     @Override

tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java

@@ -21,7 +21,6 @@ import android.test.suitebuilder.annotation.LargeTest;
 import android.util.Log;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -36,9 +35,6 @@ import java.util.Random;
 public class SparseTableTests extends AndroidTestCase {
     private static final String TAG = SparseTableTests.class.getSimpleName();
 
-    private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 };
-    private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7};
-
     private final Random mRandom;
     private final ArrayList<Integer> mRandomIndex;
 
@@ -59,26 +55,15 @@ public class SparseTableTests extends AndroidTestCase {
         }
     }
 
-    public void testInitializeWithArray() {
-        final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE);
-        for (int i = 0; i < 8; ++i) {
-            assertEquals(SparseTable.NOT_EXIST, table.get(i));
-        }
-        assertEquals(SparseTable.NOT_EXIST, table.get(8));
-        for (int i = 9; i < 16; ++i) {
-            assertEquals(i - 8, table.get(i));
-        }
-    }
-
     public void testSet() {
-        final SparseTable table = new SparseTable(16, BLOCK_SIZE);
+        final SparseTable table = new SparseTable(16, BLOCK_SIZE, 1);
-        table.set(3, 6);
+        table.set(0, 3, 6);
-        table.set(8, 16);
+        table.set(0, 8, 16);
         for (int i = 0; i < 16; ++i) {
             if (i == 3 || i == 8) {
-                assertEquals(i * 2, table.get(i));
+                assertEquals(i * 2, table.get(0, i));
             } else {
-                assertEquals(SparseTable.NOT_EXIST, table.get(i));
+                assertEquals(SparseTable.NOT_EXIST, table.get(0, i));
             }
         }
     }
@@ -94,11 +79,11 @@ public class SparseTableTests extends AndroidTestCase {
     }
 
     private void runTestRandomSet() {
-        final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE);
+        final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, 1);
         int elementCount = 0;
         for (int i = 0; i < DEFAULT_SIZE; ++i) {
             if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
-                table.set(i, mRandomIndex.get(i));
+                table.set(0, i, mRandomIndex.get(i));
                 elementCount++;
             }
         }
@@ -107,7 +92,7 @@ public class SparseTableTests extends AndroidTestCase {
               + table.getContentTableSize());
         Log.d(TAG, "the table has " + elementCount + " elements");
         for (int i = 0; i < DEFAULT_SIZE; ++i) {
-            assertEquals(table.get(i), (int)mRandomIndex.get(i));
+            assertEquals(table.get(0, i), (int)mRandomIndex.get(i));
         }
 
         // flush and reload
@@ -120,16 +105,13 @@ public class SparseTableTests extends AndroidTestCase {
             final File contentFile = File.createTempFile("testRandomSet", ".big");
             lookupOutStream = new FileOutputStream(lookupIndexFile);
             contentOutStream = new FileOutputStream(contentFile);
-            table.write(lookupOutStream, contentOutStream);
+            table.write(lookupOutStream, new OutputStream[] { contentOutStream });
-            lookupInStream = new FileInputStream(lookupIndexFile);
+            lookupOutStream.flush();
-            contentInStream = new FileInputStream(contentFile);
+            contentOutStream.flush();
-            final byte[] lookupArray = new byte[(int) lookupIndexFile.length()];
+            final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile,
-            final byte[] contentArray = new byte[(int) contentFile.length()];
+                    new File[] { contentFile }, BLOCK_SIZE);
-            lookupInStream.read(lookupArray);
-            contentInStream.read(contentArray);
-            final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE);
             for (int i = 0; i < DEFAULT_SIZE; ++i) {
-                assertEquals(table.get(i), newTable.get(i));
+                assertEquals(table.get(0, i), newTable.get(0, i));
             }
         } catch (IOException e) {
             Log.d(TAG, "IOException while flushing and realoding", e);
@@ -157,4 +139,60 @@ public class SparseTableTests extends AndroidTestCase {
             runTestRandomSet();
         }
     }
+
+    public void testMultipleContents() {
+        final int numOfContents = 5;
+        generateRandomIndex(DEFAULT_SIZE, 20);
+        final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, numOfContents);
+        for (int i = 0; i < mRandomIndex.size(); ++i) {
+            if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
+                for (int j = 0; j < numOfContents; ++j) {
+                    table.set(j, i, mRandomIndex.get(i));
+                }
+            }
+        }
+
+        OutputStream lookupOutStream = null;
+        OutputStream[] contentsOutStream = new OutputStream[numOfContents];
+        try {
+            final File lookupIndexFile = File.createTempFile("testMultipleContents", "small");
+            lookupOutStream = new FileOutputStream(lookupIndexFile);
+            final File[] contentFiles = new File[numOfContents];
+            for (int i = 0; i < numOfContents; ++i) {
+                contentFiles[i] = File.createTempFile("testMultipleContents", "big" + i);
+                contentsOutStream[i] = new FileOutputStream(contentFiles[i]);
+            }
+            table.write(lookupOutStream, contentsOutStream);
+            lookupOutStream.flush();
+            for (int i = 0; i < numOfContents; ++i) {
+                contentsOutStream[i].flush();
+            }
+            final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, contentFiles,
+                    BLOCK_SIZE);
+            for (int i = 0; i < numOfContents; ++i) {
+                for (int j = 0; j < DEFAULT_SIZE; ++j) {
+                    assertEquals(table.get(i, j), newTable.get(i, j));
+                }
+            }
+        } catch (IOException e) {
+            Log.d(TAG, "IOException while flushing and reloading", e);
+        } finally {
+            if (lookupOutStream != null) {
+                try {
+                    lookupOutStream.close();
+                } catch (IOException e) {
+                    Log.d(TAG, "IOException while closing the stream", e);
+                }
+            }
+            for (int i = 0; i < numOfContents; ++i) {
+                if (contentsOutStream[i] != null) {
+                    try {
+                        contentsOutStream[i].close();
+                    } catch (IOException e) {
+                        Log.d(TAG, "IOException while closing the stream.", e);
+                    }
+                }
+            }
+        }
+    }
 }