From ab6a93773ba3cbe93002bc37b6b61f874fc09144 Mon Sep 17 00:00:00 2001 From: Keisuke Kuroyanagi Date: Thu, 6 Feb 2014 21:55:37 +0900 Subject: [PATCH] Use native logic to read Ver4 dict. Bug: 11281877 Bug: 12810574 Change-Id: Ief371d3ef61818e4e031de4659aee3c9584c7379 --- .../latin/makedict/AbstractDictDecoder.java | 46 ++ .../latin/makedict/FusionDictionary.java | 22 +- .../latin/makedict/SparseTable.java | 223 --------- .../makedict/SparseTableContentReader.java | 126 ----- .../makedict/SparseTableContentWriter.java | 93 ---- .../latin/makedict/Ver4DictDecoder.java | 450 ++---------------- .../BinaryDictDecoderEncoderTests.java | 17 - .../latin/makedict/SparseTableTests.java | 195 -------- 8 files changed, 106 insertions(+), 1066 deletions(-) delete mode 100644 java/src/com/android/inputmethod/latin/makedict/SparseTable.java delete mode 100644 java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java delete mode 100644 java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java delete mode 100644 tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index 5c7c4b8e3..e7d1c98a9 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -22,6 +22,7 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -223,4 +224,49 @@ public abstract class AbstractDictDecoder implements DictDecoder { public boolean hasValidRawBinaryDictionary() { return checkHeader() == SUCCESS; } + + // Placeholder implementations below. These are actually unused. + @Override + public void openDictBuffer() throws FileNotFoundException, IOException, + UnsupportedFormatException { + } + + @Override + public boolean isDictBufferOpen() { + return false; + } + + @Override + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + return null; + } + + @Override + public void setPosition(int newPos) { + } + + @Override + public int getPosition() { + return 0; + } + + @Override + public int readPtNodeCount() { + return 0; + } + + @Override + public boolean readAndFollowForwardLink() { + return false; + } + + @Override + public boolean hasNextPtNodeArray() { + return false; + } + + @Override + @UsedForTesting + public void skipPtNode(final FormatOptions formatOptions) { + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index e9561afd3..ca4a2e9bb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -436,25 +436,25 @@ public final class FusionDictionary implements Iterable { /** * Helper method to add a new bigram to the dictionary. * - * @param word1 the previous word of the context - * @param word2 the next word of the context + * @param word0 the previous word of the context + * @param word1 the next word of the context * @param frequency the bigram frequency */ - public void setBigram(final String word1, final String word2, final int frequency) { - PtNode ptNode = findWordInTree(mRootNodeArray, word1); - if (ptNode != null) { - final PtNode ptNode2 = findWordInTree(mRootNodeArray, word2); - if (ptNode2 == null) { - add(getCodePoints(word2), 0, null, false /* isNotAWord */, + public void setBigram(final String word0, final String word1, final int frequency) { + PtNode ptNode0 = findWordInTree(mRootNodeArray, word0); + if (ptNode0 != null) { + final PtNode ptNode1 = findWordInTree(mRootNodeArray, word1); + if (ptNode1 == null) { + add(getCodePoints(word1), 0, null, false /* isNotAWord */, false /* isBlacklistEntry */); // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been // a cutting point until now. In this case, we need to refresh ptNode. - ptNode = findWordInTree(mRootNodeArray, word1); + ptNode0 = findWordInTree(mRootNodeArray, word0); } - ptNode.addBigram(word2, frequency); + ptNode0.addBigram(word1, frequency); } else { - throw new RuntimeException("First word of bigram not found"); + throw new RuntimeException("First word of bigram not found " + word0); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java deleted file mode 100644 index 7592a0c13..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collections; - -/** - * SparseTable is an extensible map from integer to integer. - * This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index - * memory. - */ -@UsedForTesting -public class SparseTable { - - /** - * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize - * terminals. - * It contains at index i = j / mBlockSize the index in each ArrayList in mContentsTables where - * the values for terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized - * integer array. - */ - private final ArrayList mLookupTable; - private final ArrayList> mContentTables; - - private final int mBlockSize; - private final int mContentTableCount; - public static final int NOT_EXIST = -1; - public static final int SIZE_OF_INT_IN_BYTES = 4; - - @UsedForTesting - public SparseTable(final int initialCapacity, final int blockSize, - final int contentTableCount) { - mBlockSize = blockSize; - final int lookupTableSize = initialCapacity / mBlockSize - + (initialCapacity % mBlockSize > 0 ? 1 : 0); - mLookupTable = new ArrayList(Collections.nCopies(lookupTableSize, NOT_EXIST)); - mContentTableCount = contentTableCount; - mContentTables = CollectionUtils.newArrayList(); - for (int i = 0; i < mContentTableCount; ++i) { - mContentTables.add(new ArrayList()); - } - } - - @UsedForTesting - public SparseTable(final ArrayList lookupTable, - final ArrayList> contentTables, final int blockSize) { - mBlockSize = blockSize; - mContentTableCount = contentTables.size(); - mLookupTable = lookupTable; - mContentTables = contentTables; - } - - /** - * Converts an byte array to an int array considering each set of 4 bytes is an int stored in - * big-endian. - * The length of byteArray must be a multiple of four. - * Otherwise, IndexOutOfBoundsException will be raised. - */ - @UsedForTesting - private static ArrayList convertByteArrayToIntegerArray(final byte[] byteArray) { - final ArrayList integerArray = new ArrayList(byteArray.length / 4); - for (int i = 0; i < byteArray.length; i += 4) { - int value = 0; - for (int j = i; j < i + 4; ++j) { - value <<= 8; - value |= byteArray[j] & 0xFF; - } - integerArray.add(value); - } - return integerArray; - } - - @UsedForTesting - public int get(final int contentTableIndex, final int index) { - if (!contains(index)) { - return NOT_EXIST; - } - return mContentTables.get(contentTableIndex).get( - mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); - } - - @UsedForTesting - public ArrayList getAll(final int index) { - final ArrayList ret = CollectionUtils.newArrayList(); - for (int i = 0; i < mContentTableCount; ++i) { - ret.add(get(i, index)); - } - return ret; - } - - @UsedForTesting - public void set(final int contentTableIndex, final int index, final int value) { - if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - mLookupTable.set(index / mBlockSize, mContentTables.get(contentTableIndex).size()); - for (int i = 0; i < mContentTableCount; ++i) { - for (int j = 0; j < mBlockSize; ++j) { - mContentTables.get(i).add(NOT_EXIST); - } - } - } - mContentTables.get(contentTableIndex).set( - mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); - } - - public void remove(final int indexOfContent, final int index) { - set(indexOfContent, index, NOT_EXIST); - } - - @UsedForTesting - public int size() { - return mLookupTable.size() * mBlockSize; - } - - @UsedForTesting - /* package */ int getContentTableSize() { - // This class always has at least one content table. - return mContentTables.get(0).size(); - } - - @UsedForTesting - /* package */ int getLookupTableSize() { - return mLookupTable.size(); - } - - public boolean contains(final int index) { - if (index < 0 || index / mBlockSize >= mLookupTable.size() - || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - return false; - } - return true; - } - - @UsedForTesting - public void write(final OutputStream lookupOutStream, final OutputStream[] contentOutStreams) - throws IOException { - if (contentOutStreams.length != mContentTableCount) { - throw new RuntimeException(contentOutStreams.length + " streams are given, but the" - + " table has " + mContentTableCount + " content tables."); - } - for (final int index : mLookupTable) { - BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, SIZE_OF_INT_IN_BYTES); - } - - for (int i = 0; i < contentOutStreams.length; ++i) { - for (final int data : mContentTables.get(i)) { - BinaryDictEncoderUtils.writeUIntToStream(contentOutStreams[i], data, - SIZE_OF_INT_IN_BYTES); - } - } - } - - @UsedForTesting - public void writeToFiles(final File lookupTableFile, final File[] contentFiles) - throws IOException { - FileOutputStream lookupTableOutStream = null; - final FileOutputStream[] contentTableOutStreams = new FileOutputStream[mContentTableCount]; - try { - lookupTableOutStream = new FileOutputStream(lookupTableFile); - for (int i = 0; i < contentFiles.length; ++i) { - contentTableOutStreams[i] = new FileOutputStream(contentFiles[i]); - } - write(lookupTableOutStream, contentTableOutStreams); - } finally { - if (lookupTableOutStream != null) { - lookupTableOutStream.close(); - } - for (int i = 0; i < contentTableOutStreams.length; ++i) { - if (contentTableOutStreams[i] != null) { - contentTableOutStreams[i].close(); - } - } - } - } - - private static byte[] readFileToByteArray(final File file) throws IOException { - final byte[] contents = new byte[(int) file.length()]; - FileInputStream inStream = null; - try { - inStream = new FileInputStream(file); - inStream.read(contents); - } finally { - if (inStream != null) { - inStream.close(); - } - } - return contents; - } - - @UsedForTesting - public static SparseTable readFromFiles(final File lookupTableFile, final File[] contentFiles, - final int blockSize) throws IOException { - final ArrayList> contentTables = - new ArrayList>(contentFiles.length); - for (int i = 0; i < contentFiles.length; ++i) { - contentTables.add(convertByteArrayToIntegerArray(readFileToByteArray(contentFiles[i]))); - } - return new SparseTable(convertByteArrayToIntegerArray(readFileToByteArray(lookupTableFile)), - contentTables, blockSize); - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java deleted file mode 100644 index 63e1f56f5..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; - -/** - * An auxiliary class for reading SparseTable and data written by SparseTableContentWriter. - */ -public class SparseTableContentReader { - - /** - * An interface of a function which is passed to SparseTableContentReader.read. - */ - public interface SparseTableContentReaderInterface { - /** - * Reads data. - * - * @param buffer the DictBuffer. The position of the buffer is set to the head of data. - */ - public void read(final DictBuffer buffer); - } - - protected final int mContentCount; - protected final int mBlockSize; - protected final File mBaseDir; - protected final File mLookupTableFile; - protected final File[] mAddressTableFiles; - protected final File[] mContentFiles; - protected DictBuffer mLookupTableBuffer; - protected final DictBuffer[] mAddressTableBuffers; - private final DictBuffer[] mContentBuffers; - protected final DictionaryBufferFactory mFactory; - - /** - * Sole constructor of SparseTableContentReader. - * - * @param name the name of SparseTable. - * @param blockSize the block size of the content table. - * @param baseDir the directory which contains the files of the content table. - * @param contentFilenames the file names of content files. - * @param contentSuffixes the ids of contents. These ids are used for a suffix of a name of - * address files and content files. - * @param factory the DictionaryBufferFactory which is used for opening the files. - */ - public SparseTableContentReader(final String name, final int blockSize, final File baseDir, - final String[] contentFilenames, final String[] contentSuffixes, - final DictionaryBufferFactory factory) { - if (contentFilenames.length != contentSuffixes.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentSuffixes are different " + contentFilenames.length + ", " - + contentSuffixes.length); - } - mBlockSize = blockSize; - mBaseDir = baseDir; - mFactory = factory; - mContentCount = contentFilenames.length; - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentSuffixes[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentSuffixes[i]); - } - mAddressTableBuffers = new DictBuffer[mContentCount]; - mContentBuffers = new DictBuffer[mContentCount]; - } - - public void openBuffers() throws FileNotFoundException, IOException { - mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); - for (int i = 0; i < mContentCount; ++i) { - mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); - mContentBuffers[i] = mFactory.getDictionaryBuffer(mContentFiles[i]); - } - } - - /** - * Calls the read() callback of the reader with the appropriate buffer appropriately positioned. - * @param contentNumber the index in the original contentFilenames[] array. - * @param terminalId the terminal ID to read. - * @param reader the reader on which to call the callback. - */ - protected void read(final int contentNumber, final int terminalId, - final SparseTableContentReaderInterface reader) { - if (terminalId < 0 || (terminalId / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES - >= mLookupTableBuffer.limit()) { - return; - } - - mLookupTableBuffer.position((terminalId / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - final int indexInAddressTable = mLookupTableBuffer.readInt(); - if (indexInAddressTable == SparseTable.NOT_EXIST) { - return; - } - - mAddressTableBuffers[contentNumber].position(SparseTable.SIZE_OF_INT_IN_BYTES - * ((indexInAddressTable * mBlockSize) + (terminalId % mBlockSize))); - final int address = mAddressTableBuffers[contentNumber].readInt(); - if (address == SparseTable.NOT_EXIST) { - return; - } - - mContentBuffers[contentNumber].position(address); - reader.read(mContentBuffers[contentNumber]); - } -} \ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java deleted file mode 100644 index 49f0fd624..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * An auxiliary class for writing data associated with SparseTable to files. - */ -public class SparseTableContentWriter { - public interface SparseTableContentWriterInterface { - public void write(final OutputStream outStream) throws IOException; - } - - private final int mContentCount; - private final SparseTable mSparseTable; - private final File mLookupTableFile; - protected final File mBaseDir; - private final File[] mAddressTableFiles; - private final File[] mContentFiles; - protected final OutputStream[] mContentOutStreams; - - /** - * Sole constructor of SparseTableContentWriter. - * - * @param name the name of SparseTable. - * @param initialCapacity the initial capacity of SparseTable. - * @param blockSize the block size of the content table. - * @param baseDir the directory which contains the files of the content table. - * @param contentFilenames the file names of content files. - * @param contentIds the ids of contents. These ids are used for a suffix of a name of address - * files and content files. - */ - public SparseTableContentWriter(final String name, final int initialCapacity, - final int blockSize, final File baseDir, final String[] contentFilenames, - final String[] contentIds) { - if (contentFilenames.length != contentIds.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); - } - mContentCount = contentFilenames.length; - mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - mBaseDir = baseDir; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); - } - mContentOutStreams = new OutputStream[mContentCount]; - } - - public void openStreams() throws FileNotFoundException { - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); - } - } - - protected void write(final int contentIndex, final int index, - final SparseTableContentWriterInterface writer) throws IOException { - mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); - writer.write(mContentOutStreams[contentIndex]); - mContentOutStreams[contentIndex].flush(); - } - - public void closeStreams() throws IOException { - mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i].close(); - } - } -} \ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 9ddaaf734..83707480d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -17,20 +17,15 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; - -import android.util.Log; +import com.android.inputmethod.latin.utils.FileUtils; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; /** * An implementation of binary dictionary decoder for version 4 binary dictionary. @@ -39,421 +34,74 @@ import java.util.Arrays; public class Ver4DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver4DictDecoder.class.getSimpleName(); - protected static final int FILETYPE_TRIE = 1; - protected static final int FILETYPE_FREQUENCY = 2; - protected static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; - protected static final int FILETYPE_BIGRAM_FREQ = 4; - protected static final int FILETYPE_SHORTCUT = 5; - protected static final int FILETYPE_HEADER = 6; - - protected final File mDictDirectory; - protected final DictionaryBufferFactory mBufferFactory; - protected DictBuffer mDictBuffer; - protected DictBuffer mHeaderBuffer; - protected DictBuffer mFrequencyBuffer; - protected DictBuffer mTerminalAddressTableBuffer; - private BigramContentReader mBigramReader; - private ShortcutContentReader mShortcutReader; - - /** - * Raw PtNode info straight out of a trie file in version 4 dictionary. - */ - protected static final class Ver4PtNodeInfo { - public final int mFlags; - public final int[] mCharacters; - public final int mTerminalId; - public final int mChildrenPos; - public final int mParentPos; - public final int mNodeSize; - public int mStartIndexOfCharacters; - public int mEndIndexOfCharacters; // exclusive - - public Ver4PtNodeInfo(final int flags, final int[] characters, final int terminalId, - final int childrenPos, final int parentPos, final int nodeSize) { - mFlags = flags; - mCharacters = characters; - mTerminalId = terminalId; - mChildrenPos = childrenPos; - mParentPos = parentPos; - mNodeSize = nodeSize; - mStartIndexOfCharacters = 0; - mEndIndexOfCharacters = characters.length; - } - } + final File mDictDirectory; + final BinaryDictionary mBinaryDictionary; @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { - mDictDirectory = dictDirectory; - mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null; - - if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { - mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); - } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { - mBufferFactory = new DictionaryBufferFromByteArrayFactory(); - } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { - mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); - } else { - mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); - } + this(dictDirectory, null /* factory */); } @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) { mDictDirectory = dictDirectory; - mBufferFactory = factory; - mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null; - } - - protected File getFile(final int fileType) throws UnsupportedFormatException { - if (fileType == FILETYPE_TRIE) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION); - } else if (fileType == FILETYPE_HEADER) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.HEADER_FILE_EXTENSION); - } else if (fileType == FILETYPE_FREQUENCY) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION); - } else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); - } else if (fileType == FILETYPE_BIGRAM_FREQ) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION - + FormatSpec.BIGRAM_FREQ_CONTENT_ID); - } else if (fileType == FILETYPE_SHORTCUT) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION - + FormatSpec.SHORTCUT_CONTENT_ID); - } else { - throw new UnsupportedFormatException("Unsupported kind of file : " + fileType); - } - } - - @Override - public void openDictBuffer() throws FileNotFoundException, IOException, - UnsupportedFormatException { - if (!mDictDirectory.isDirectory()) { - throw new UnsupportedFormatException("Format 4 dictionary needs a directory"); - } - mHeaderBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_HEADER)); - mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE)); - mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); - mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( - getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); - mBigramReader = new BigramContentReader(mDictDirectory.getName(), - mDictDirectory, mBufferFactory); - mBigramReader.openBuffers(); - mShortcutReader = new ShortcutContentReader(mDictDirectory.getName(), mDictDirectory, - mBufferFactory); - mShortcutReader.openBuffers(); - } - - @Override - public boolean isDictBufferOpen() { - return mDictBuffer != null; - } - - @UsedForTesting - /* package */ DictBuffer getHeaderBuffer() { - return mHeaderBuffer; - } - - @UsedForTesting - /* package */ DictBuffer getDictBuffer() { - return mDictBuffer; + mBinaryDictionary = new BinaryDictionary(dictDirectory.getAbsolutePath(), + 0 /* offset */, 0 /* length */, true /* useFullEditDistance */, null /* locale */, + "" /* dictType */, true /* isUpdatable */); } @Override public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { - if (mHeaderBuffer == null) { - openDictBuffer(); - } - mHeaderBuffer.position(0); - final DictionaryHeader header = super.readHeader(mHeaderBuffer); - final int version = header.mFormatOptions.mVersion; - if (version != FormatSpec.VERSION4) { - throw new UnsupportedFormatException("File header has a wrong version : " + version); - } - return header; - } - - /** - * An auxiliary class for reading bigrams. - */ - protected static class BigramContentReader extends SparseTableContentReader { - public BigramContentReader(final String name, final File baseDir, - final DictionaryBufferFactory factory) { - super(name + FormatSpec.BIGRAM_FILE_EXTENSION, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - getContentFilenames(name), getContentIds(), factory); - } - - // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. - protected static String[] getContentFilenames(final String name) { - return new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }; - } - - // TODO: Consolidate this method and BigramContentWriter.getContentIds. - protected static String[] getContentIds() { - return new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }; - } - - public ArrayList readTargetsAndFrequencies(final int terminalId, - final DictBuffer terminalAddressTableBuffer, final FormatOptions options) { - final ArrayList bigrams = CollectionUtils.newArrayList(); - read(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, - new SparseTableContentReaderInterface() { - @Override - public void read(final DictBuffer buffer) { - while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, - // remaining bigram entries are ignored. - final int bigramFlags = buffer.readUnsignedByte(); - final int probability; - - if (options.mHasTimestamp) { - probability = buffer.readUnsignedByte(); - // Skip timestamp - buffer.readInt(); - // Skip level - buffer.readUnsignedByte(); - // Skip count - buffer.readUnsignedByte(); - } else { - probability = bigramFlags - & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; - } - final int targetTerminalId = buffer.readUnsignedInt24(); - terminalAddressTableBuffer.position(targetTerminalId - * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - final int targetAddress = - terminalAddressTableBuffer.readUnsignedInt24(); - bigrams.add(new PendingAttribute(probability, targetAddress)); - if (0 == (bigramFlags - & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { - break; - } - } - if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode (" - + bigrams.size() + " but max is " - + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); - } - } - }); - if (bigrams.isEmpty()) return null; - return bigrams; - } - } - - /** - * An auxiliary class for reading shortcuts. - */ - protected static class ShortcutContentReader extends SparseTableContentReader { - public ShortcutContentReader(final String name, final File baseDir, - final DictionaryBufferFactory factory) { - super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, - FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, - new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, factory); - } - - public ArrayList readShortcuts(final int terminalId) { - final ArrayList shortcuts = CollectionUtils.newArrayList(); - read(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, - new SparseTableContentReaderInterface() { - @Override - public void read(final DictBuffer buffer) { - while (true) { - final int flags = buffer.readUnsignedByte(); - final String word = CharEncoding.readString(buffer); - shortcuts.add(new WeightedString(word, - flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { - break; - } - } - } - }); - if (shortcuts.isEmpty()) return null; - return shortcuts; - } - } - - protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { - protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId, - final FormatOptions formatOptions) { - final int readingPos; - if (formatOptions.mHasTimestamp) { - final int entrySize = FormatSpec.FREQUENCY_AND_FLAGS_SIZE - + FormatSpec.UNIGRAM_TIMESTAMP_SIZE + FormatSpec.UNIGRAM_LEVEL_SIZE - + FormatSpec.UNIGRAM_COUNTER_SIZE; - readingPos = terminalId * entrySize + FormatSpec.FLAGS_IN_FREQ_FILE_SIZE; - } else { - readingPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE - + FormatSpec.FLAGS_IN_FREQ_FILE_SIZE; - } - frequencyBuffer.position(readingPos); - return frequencyBuffer.readUnsignedByte(); - } - - protected static int readTerminalId(final DictBuffer dictBuffer) { - return dictBuffer.readInt(); - } - } - - private final int[] mCharacterBufferForReadingVer4PtNodeInfo - = new int[FormatSpec.MAX_WORD_LENGTH]; - - /** - * Reads PtNode from ptNodePos in the trie file and returns Ver4PtNodeInfo. - * - * @param ptNodePos the position of PtNode. - * @param options the format options. - * @return Ver4PtNodeInfo. - */ - // TODO: Make this buffer thread safe. - // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. - protected Ver4PtNodeInfo readVer4PtNodeInfo(final int ptNodePos, final FormatOptions options) { - int readingPos = ptNodePos; - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - readingPos += FormatSpec.PTNODE_FLAGS_SIZE; - - final int parentPos = PtNodeReader.readParentAddress(mDictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - readingPos += FormatSpec.PARENT_ADDRESS_SIZE; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); - while (FormatSpec.INVALID_CHARACTER != character - && index < FormatSpec.MAX_WORD_LENGTH) { - mCharacterBufferForReadingVer4PtNodeInfo[index++] = character; - character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(mCharacterBufferForReadingVer4PtNodeInfo, 0, index); - } else { - final int character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int terminalId; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - terminalId = PtNodeReader.readTerminalId(mDictBuffer); - readingPos += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } else { - terminalId = PtNode.NOT_A_TERMINAL; - } - - int childrenPos = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); - if (childrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenPos += readingPos; - } - readingPos += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - - return new Ver4PtNodeInfo(flags, characters, terminalId, childrenPos, parentPos, - readingPos - ptNodePos); - } - - @Override - public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { - final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(ptNodePos, options); - - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & nodeInfo.mFlags)) { - frequency = PtNodeReader.readFrequency(mFrequencyBuffer, nodeInfo.mTerminalId, options); - } else { - frequency = PtNode.NOT_A_TERMINAL; - } - - final ArrayList shortcutTargets = mShortcutReader.readShortcuts( - nodeInfo.mTerminalId); - final ArrayList bigrams = mBigramReader.readTargetsAndFrequencies( - nodeInfo.mTerminalId, mTerminalAddressTableBuffer, options); - - return new PtNodeInfo(ptNodePos, ptNodePos + nodeInfo.mNodeSize, nodeInfo.mFlags, - nodeInfo.mCharacters, frequency, nodeInfo.mParentPos, nodeInfo.mChildrenPos, - shortcutTargets, bigrams); - } - - private void deleteDictFiles() { - final File[] files = mDictDirectory.listFiles(); - for (int i = 0; i < files.length; ++i) { - files[i].delete(); - } + return mBinaryDictionary.getHeader(); } @Override public FusionDictionary readDictionaryBinary(final FusionDictionary dict, final boolean deleteDictIfBroken) throws FileNotFoundException, IOException, UnsupportedFormatException { - if (mDictBuffer == null) { - openDictBuffer(); - } - try { - return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); - } catch (IOException e) { - Log.e(TAG, "The dictionary " + mDictDirectory.getName() + " is broken.", e); - if (deleteDictIfBroken) { - deleteDictFiles(); + final DictionaryHeader header = readHeader(); + final FusionDictionary fusionDict = dict != null ? dict : + new FusionDictionary(new FusionDictionary.PtNodeArray(), header.mDictionaryOptions); + int token = 0; + final ArrayList wordProperties = CollectionUtils.newArrayList(); + do { + final BinaryDictionary.GetNextWordPropertyResult result = + mBinaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + if (wordProperty == null) { + if (deleteDictIfBroken) { + mBinaryDictionary.close(); + FileUtils.deleteRecursively(mDictDirectory); + } + return null; } - throw e; - } catch (UnsupportedFormatException e) { - Log.e(TAG, "The dictionary " + mDictDirectory.getName() + " is broken.", e); - if (deleteDictIfBroken) { - deleteDictFiles(); - } - throw e; - } - } + wordProperties.add(wordProperty); + token = result.mNextToken; + } while (token != 0); - @Override - public void setPosition(int newPos) { - mDictBuffer.position(newPos); - } - - @Override - public int getPosition() { - return mDictBuffer.position(); - } - - @Override - public int readPtNodeCount() { - return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); - } - - @Override - public boolean readAndFollowForwardLink() { - final int forwardLinkPos = mDictBuffer.position(); - int nextRelativePos = BinaryDictDecoderUtils.readSInt24(mDictBuffer); - if (nextRelativePos != FormatSpec.NO_FORWARD_LINK_ADDRESS) { - final int nextPos = forwardLinkPos + nextRelativePos; - if (nextPos >= 0 && nextPos < mDictBuffer.limit()) { - mDictBuffer.position(nextPos); - return true; + // Insert unigrams to the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + // TODO: Support probability that is -1. + final int probability = wordProperty.getProbability() < 0 ? + 0 : wordProperty.getProbability(); + if (wordProperty.mIsBlacklistEntry) { + fusionDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets, + wordProperty.mIsNotAWord); + } else { + fusionDict.add(wordProperty.mWord, probability, + wordProperty.mShortcutTargets, wordProperty.mIsNotAWord); } } - return false; - } - - @Override - public boolean hasNextPtNodeArray() { - return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; - } - - @Override - @UsedForTesting - public void skipPtNode(final FormatOptions formatOptions) { - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - PtNodeReader.readParentAddress(mDictBuffer, formatOptions); - BinaryDictIOUtils.skipString(mDictBuffer, - (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); - PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); + // Insert bigrams to the fusion dictionary. + for (final WordProperty wordProperty : wordProperties) { + if (wordProperty.mBigrams == null) { + continue; + } + final String word0 = wordProperty.mWord; + for (final WeightedString bigram : wordProperty.mBigrams) { + fusionDict.setBigram(word0, bigram.mWord, bigram.getProbability()); + } + } + return fusionDict; } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 0815819d6..bc4452fd6 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -468,10 +468,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2_OPTIONS); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -483,10 +479,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2_OPTIONS); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -597,17 +589,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2_OPTIONS); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, - BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2_OPTIONS); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_OPTIONS_WITHOUT_TIMESTAMP); - runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, - BinaryDictUtils.VERSION4_OPTIONS_WITH_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java deleted file mode 100644 index aeb8552bd..000000000 --- a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.LargeTest; -import android.util.Log; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Random; - -/** - * Unit tests for SparseTable. - */ -@LargeTest -public class SparseTableTests extends AndroidTestCase { - private static final String TAG = SparseTableTests.class.getSimpleName(); - - private final Random mRandom; - private final ArrayList mRandomIndex; - - private static final int DEFAULT_SIZE = 10000; - private static final int BLOCK_SIZE = 8; - - public SparseTableTests() { - this(System.currentTimeMillis(), DEFAULT_SIZE); - } - - public SparseTableTests(final long seed, final int tableSize) { - super(); - Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize); - mRandom = new Random(seed); - mRandomIndex = new ArrayList(tableSize); - for (int i = 0; i < tableSize; ++i) { - mRandomIndex.add(SparseTable.NOT_EXIST); - } - } - - public void testSet() { - final SparseTable table = new SparseTable(16, BLOCK_SIZE, 1); - table.set(0, 3, 6); - table.set(0, 8, 16); - for (int i = 0; i < 16; ++i) { - if (i == 3 || i == 8) { - assertEquals(i * 2, table.get(0, i)); - } else { - assertEquals(SparseTable.NOT_EXIST, table.get(0, i)); - } - } - } - - private void generateRandomIndex(final int size, final int prop) { - for (int i = 0; i < size; ++i) { - if (mRandom.nextInt(100) < prop) { - mRandomIndex.set(i, mRandom.nextInt()); - } else { - mRandomIndex.set(i, SparseTable.NOT_EXIST); - } - } - } - - private void runTestRandomSet() { - final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, 1); - int elementCount = 0; - for (int i = 0; i < DEFAULT_SIZE; ++i) { - if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { - table.set(0, i, mRandomIndex.get(i)); - elementCount++; - } - } - - Log.d(TAG, "table size = " + table.getLookupTableSize() + " + " - + table.getContentTableSize()); - Log.d(TAG, "the table has " + elementCount + " elements"); - for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(0, i), (int)mRandomIndex.get(i)); - } - - // flush and reload - OutputStream lookupOutStream = null; - OutputStream contentOutStream = null; - try { - final File lookupIndexFile = File.createTempFile("testRandomSet", ".small"); - final File contentFile = File.createTempFile("testRandomSet", ".big"); - lookupOutStream = new FileOutputStream(lookupIndexFile); - contentOutStream = new FileOutputStream(contentFile); - table.write(lookupOutStream, new OutputStream[] { contentOutStream }); - lookupOutStream.flush(); - contentOutStream.flush(); - final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, - new File[] { contentFile }, BLOCK_SIZE); - for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(0, i), newTable.get(0, i)); - } - } catch (IOException e) { - Log.d(TAG, "IOException while flushing and realoding", e); - } finally { - if (lookupOutStream != null) { - try { - lookupOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream", e); - } - } - if (contentOutStream != null) { - try { - contentOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing contentStream.", e); - } - } - } - } - - public void testRandomSet() { - for (int i = 0; i <= 100; i += 10) { - generateRandomIndex(DEFAULT_SIZE, i); - runTestRandomSet(); - } - } - - public void testMultipleContents() { - final int numOfContents = 5; - generateRandomIndex(DEFAULT_SIZE, 20); - final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, numOfContents); - for (int i = 0; i < mRandomIndex.size(); ++i) { - if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { - for (int j = 0; j < numOfContents; ++j) { - table.set(j, i, mRandomIndex.get(i)); - } - } - } - - OutputStream lookupOutStream = null; - OutputStream[] contentsOutStream = new OutputStream[numOfContents]; - try { - final File lookupIndexFile = File.createTempFile("testMultipleContents", "small"); - lookupOutStream = new FileOutputStream(lookupIndexFile); - final File[] contentFiles = new File[numOfContents]; - for (int i = 0; i < numOfContents; ++i) { - contentFiles[i] = File.createTempFile("testMultipleContents", "big" + i); - contentsOutStream[i] = new FileOutputStream(contentFiles[i]); - } - table.write(lookupOutStream, contentsOutStream); - lookupOutStream.flush(); - for (int i = 0; i < numOfContents; ++i) { - contentsOutStream[i].flush(); - } - final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, contentFiles, - BLOCK_SIZE); - for (int i = 0; i < numOfContents; ++i) { - for (int j = 0; j < DEFAULT_SIZE; ++j) { - assertEquals(table.get(i, j), newTable.get(i, j)); - } - } - } catch (IOException e) { - Log.d(TAG, "IOException while flushing and reloading", e); - } finally { - if (lookupOutStream != null) { - try { - lookupOutStream.close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream", e); - } - } - for (int i = 0; i < numOfContents; ++i) { - if (contentsOutStream[i] != null) { - try { - contentsOutStream[i].close(); - } catch (IOException e) { - Log.d(TAG, "IOException while closing the stream.", e); - } - } - } - } - } -}