From 3feacba1eb43fb906f50581df19ff2e468642ca8 Mon Sep 17 00:00:00 2001 From: Yuichiro Hanada Date: Mon, 12 Aug 2013 14:47:20 +0900 Subject: [PATCH] Add BinaryDictReader. Bug: 9618601 Change-Id: Ief07fa0c3c4f7f5999a3fafcef4e47b6b6fd8143 --- .../latin/makedict/BinaryDictIOUtils.java | 10 +- .../latin/makedict/BinaryDictInputOutput.java | 20 ++- .../latin/makedict/BinaryDictReader.java | 109 ++++++++++++++ .../DynamicPredictionDictionaryBase.java | 21 +-- .../latin/utils/UserHistoryDictIOUtils.java | 6 +- .../latin/makedict/BinaryDictIOTests.java | 70 ++++----- .../latin/makedict/BinaryDictReaderTests.java | 138 ++++++++++++++++++ .../utils/UserHistoryDictIOUtilsTests.java | 21 +-- .../dicttool/BinaryDictOffdeviceUtils.java | 10 +- .../latin/dicttool/DictionaryMaker.java | 23 +-- .../BinaryDictOffdeviceUtilsTests.java | 9 +- 11 files changed, 327 insertions(+), 110 deletions(-) create mode 100644 java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java create mode 100644 tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 167c6915c..c7b063daf 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -145,21 +145,21 @@ public final class BinaryDictIOUtils { * Reads unigrams and bigrams from the binary file. * Doesn't make the memory representation of the dictionary. * - * @param buffer the buffer to read. + * @param reader the reader. * @param words the map to store the address as a key and the word as a value. * @param frequencies the map to store the address as a key and the frequency as a value. * @param bigrams the map to store the address as a key and the list of address as a value. * @throws IOException * @throws UnsupportedFormatException */ - public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer, + public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader, final Map words, final Map frequencies, final Map> bigrams) throws IOException, UnsupportedFormatException { // Read header - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams, - header.mFormatOptions); + final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer()); + readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words, + frequencies, bigrams, header.mFormatOptions); } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index e0874bbd5..504349a0b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -66,6 +66,7 @@ public final class BinaryDictInputOutput { public void position(int newPosition); public void put(final byte b); public int limit(); + @UsedForTesting public int capacity(); } @@ -1722,23 +1723,30 @@ public final class BinaryDictInputOutput { * FusionDictionary structure. The optional dict argument is an existing dictionary to * which words from the buffer should be added. If it is null, a new dictionary is created. * - * @param buffer the buffer to read. + * @param reader the reader. * @param dict an optional dictionary to add words to, or null. * @return the created (or merged) dictionary. */ @UsedForTesting - public static FusionDictionary readDictionaryBinary( - final FusionDictionaryBufferInterface buffer, final FusionDictionary dict) - throws IOException, UnsupportedFormatException { + public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader, + final FusionDictionary dict) throws FileNotFoundException, IOException, + UnsupportedFormatException { // clear cache wordCache.clear(); + // if the buffer has not been opened, open the buffer with bytebuffer. + if (reader.getBuffer() == null) reader.openBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + if (reader.getBuffer() == null) { + MakedictLog.e("Cannot open the buffer"); + } + // Read header - final FileHeader header = readHeader(buffer); + final FileHeader header = readHeader(reader.getBuffer()); Map reverseNodeMapping = new TreeMap(); Map reverseGroupMapping = new TreeMap(); - final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping, + final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping, reverseGroupMapping, header.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java new file mode 100644 index 000000000..57a583228 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.utils.ByteArrayWrapper; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +public class BinaryDictReader { + + public interface FusionDictionaryBufferFactory { + public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file) + throws FileNotFoundException, IOException; + } + + /** + * Creates FusionDictionaryBuffer from a ByteBuffer + */ + public static final class FusionDictionaryBufferFromByteBufferFactory + implements FusionDictionaryBufferFactory { + @Override + public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + ByteBuffer buffer = null; + try { + inStream = new FileInputStream(file); + buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.length()); + } finally { + if (inStream != null) { + inStream.close(); + } + } + if (buffer != null) { + return new BinaryDictInputOutput.ByteBufferWrapper(buffer); + } + return null; + } + } + + /** + * Creates FusionDictionaryBuffer from a byte array + */ + public static final class FusionDictionaryBufferFromByteArrayFactory + implements FusionDictionaryBufferFactory { + @Override + public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final byte[] array = new byte[(int) file.length()]; + inStream.read(array); + return new ByteArrayWrapper(array); + } finally { + if (inStream != null) { + inStream.close(); + } + } + } + } + + private final File mDictionaryBinaryFile; + private FusionDictionaryBufferInterface mFusionDictionaryBuffer; + + public BinaryDictReader(final File file) { + mDictionaryBinaryFile = file; + mFusionDictionaryBuffer = null; + } + + public void openBuffer(final FusionDictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + mFusionDictionaryBuffer = factory.getFusionDictionaryBuffer(mDictionaryBinaryFile); + } + + public FusionDictionaryBufferInterface getBuffer() { + return mFusionDictionaryBuffer; + } + + @UsedForTesting + public FusionDictionaryBufferInterface openAndGetBuffer( + final FusionDictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + openBuffer(factory); + return getBuffer(); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index a8600c09f..065e00e4a 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -28,9 +28,9 @@ import com.android.inputmethod.latin.ExpandableDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.settings.Settings; -import com.android.inputmethod.latin.utils.ByteArrayWrapper; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; @@ -39,7 +39,6 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -234,27 +233,17 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona }; // Load the dictionary from binary file - FileInputStream inStream = null; + final BinaryDictReader reader = new BinaryDictReader( + new File(getContext().getFilesDir(), fileName)); try { - final File file = new File(getContext().getFilesDir(), fileName); - final byte[] buffer = new byte[(int)file.length()]; - inStream = new FileInputStream(file); - inStream.read(buffer); - UserHistoryDictIOUtils.readDictionaryBinary( - new ByteArrayWrapper(buffer), listener); + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this // language yet. It will be created sometime later. } catch (IOException e) { Log.e(TAG, "IOException on opening a bytebuffer", e); } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } if (PROFILE_SAVE_RESTORE) { final long diff = System.currentTimeMillis() - now; Log.d(TAG, "PROF: Load UserHistoryDictionary: " diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index d02f7187e..a0ad27cfb 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -21,7 +21,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.Node; @@ -118,13 +118,13 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer, + public static void readDictionaryBinary(final BinaryDictReader reader, final OnAddWordListener dict) { final Map unigrams = CollectionUtils.newTreeMap(); final Map frequencies = CollectionUtils.newTreeMap(); final Map> bigrams = CollectionUtils.newTreeMap(); try { - BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies, + BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, unigrams, frequencies, bigrams); } catch (IOException e) { Log.e(TAG, "IO exception while reading file", e); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java index ef4ed3396..e4b5ad279 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java @@ -27,15 +27,13 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.Node; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.ByteArrayWrapper; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -120,31 +118,13 @@ public class BinaryDictIOTests extends AndroidTestCase { /** * Makes new buffer according to BUFFER_TYPE. */ - private FusionDictionaryBufferInterface getBuffer(final File file, final int bufferType) { - FileInputStream inStream = null; - try { - inStream = new FileInputStream(file); - if (bufferType == USE_BYTE_ARRAY) { - final byte[] array = new byte[(int)file.length()]; - inStream.read(array); - return new ByteArrayWrapper(array); - } else if (bufferType == USE_BYTE_BUFFER){ - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - return new BinaryDictInputOutput.ByteBufferWrapper(buffer); - } - } catch (IOException e) { - Log.e(TAG, "IOException while making buffer", e); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - Log.e(TAG, "IOException while closing stream", e); - } - } + private void getBuffer(final BinaryDictReader reader, final int bufferType) + throws FileNotFoundException, IOException { + if (bufferType == USE_BYTE_BUFFER) { + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + } else if (bufferType == USE_BYTE_ARRAY) { + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); } - return null; } /** @@ -285,13 +265,14 @@ public class BinaryDictIOTests extends AndroidTestCase { final SparseArray> bigrams, final Map> shortcutMap, final int bufferType) { long now, diff = -1; - final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType); - assertNotNull(buffer); + final BinaryDictReader reader = new BinaryDictReader(file); FusionDictionary dict = null; try { + getBuffer(reader, bufferType); + assertNotNull(reader.getBuffer()); now = System.currentTimeMillis(); - dict = BinaryDictInputOutput.readDictionaryBinary(buffer, null); + dict = BinaryDictInputOutput.readDictionaryBinary(reader, null); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); @@ -421,11 +402,12 @@ public class BinaryDictIOTests extends AndroidTestCase { final Map resultFreqs = CollectionUtils.newTreeMap(); long now = -1, diff = -1; - final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType); - assertNotNull("Can't get buffer.", buffer); + final BinaryDictReader reader = new BinaryDictReader(file); try { + getBuffer(reader, bufferType); + assertNotNull("Can't get buffer.", reader.getBuffer()); now = System.currentTimeMillis(); - BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs, + BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; } catch (IOException e) { @@ -562,7 +544,16 @@ public class BinaryDictIOTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY); + final BinaryDictReader reader = new BinaryDictReader(file); + FusionDictionaryBufferInterface buffer = null; + try { + buffer = reader.openAndGetBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + } catch (IOException e) { + // ignore + Log.e(TAG, "IOException while opening the buffer", e); + } + assertNotNull("Can't get the buffer", buffer); try { // too long word @@ -614,7 +605,16 @@ public class BinaryDictIOTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY); + final BinaryDictReader reader = new BinaryDictReader(file); + FusionDictionaryBufferInterface buffer = null; + try { + buffer = reader.openAndGetBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + } catch (IOException e) { + // ignore + Log.e(TAG, "IOException while opening the buffer", e); + } + assertNotNull("Can't get the buffer", buffer); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java new file mode 100644 index 000000000..5f6950a4e --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory; +import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory; +import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory; + +import android.test.AndroidTestCase; +import android.util.Log; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * Unit tests for BinaryDictReader + */ +public class BinaryDictReaderTests extends AndroidTestCase { + private static final String TAG = BinaryDictReaderTests.class.getSimpleName(); + + private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + + // Utilities for testing + public void writeDataToFile(final File file) { + FileOutputStream outStream = null; + try { + outStream = new FileOutputStream(file); + outStream.write(data); + } catch (IOException e) { + fail ("Can't write data to the test file"); + } finally { + if (outStream != null) { + try { + outStream.close(); + } catch (IOException e) { + Log.e(TAG, "Failed to close the output stream", e); + } + } + } + } + + @SuppressWarnings("null") + public void runTestOpenBuffer(final String testName, + final FusionDictionaryBufferFactory factory) { + File testFile = null; + try { + testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); + } catch (IOException e) { + Log.e(TAG, "IOException while the creating temporary file", e); + } + + assertNotNull(testFile); + final BinaryDictReader reader = new BinaryDictReader(testFile); + try { + reader.openBuffer(factory); + } catch (Exception e) { + Log.e(TAG, "Failed to open the buffer", e); + } + + writeDataToFile(testFile); + + try { + reader.openBuffer(factory); + } catch (Exception e) { + Log.e(TAG, "Raised the exception while opening buffer", e); + } + + assertEquals(testFile.length(), reader.getBuffer().capacity()); + } + + public void testOpenBufferWithByteBuffer() { + runTestOpenBuffer("testOpenBufferWithByteBuffer", + new FusionDictionaryBufferFromByteBufferFactory()); + } + + public void testOpenBufferWithByteArray() { + runTestOpenBuffer("testOpenBufferWithByteArray", + new FusionDictionaryBufferFromByteArrayFactory()); + } + + @SuppressWarnings("null") + public void runTestGetBuffer(final String testName, + final FusionDictionaryBufferFactory factory) { + File testFile = null; + try { + testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); + } catch (IOException e) { + Log.e(TAG, "IOException while the creating temporary file", e); + } + + final BinaryDictReader reader = new BinaryDictReader(testFile); + + // the default return value of getBuffer() must be null. + assertNull("the default return value of getBuffer() is not null", reader.getBuffer()); + + writeDataToFile(testFile); + assertTrue(testFile.exists()); + Log.d(TAG, "file length = " + testFile.length()); + + FusionDictionaryBufferInterface buffer = null; + try { + buffer = reader.openAndGetBuffer(factory); + } catch (IOException e) { + Log.e(TAG, "Failed to open and get the buffer", e); + } + assertNotNull("the buffer must not be null", buffer); + + for (int i = 0; i < data.length; ++i) { + assertEquals(data[i], buffer.readUnsignedByte()); + } + } + + public void testGetBufferWithByteBuffer() { + runTestGetBuffer("testGetBufferWithByteBuffer", + new FusionDictionaryBufferFromByteBufferFactory()); + } + + public void testGetBufferWithByteArray() { + runTestGetBuffer("testGetBufferWithByteArray", + new FusionDictionaryBufferFromByteArrayFactory()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index b6798392f..ce62bf21a 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -21,16 +21,15 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; -import com.android.inputmethod.latin.utils.ByteArrayWrapper; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -147,27 +146,15 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase } private void readDictFromFile(final File file, final OnAddWordListener listener) { - FileInputStream inStream = null; - + final BinaryDictReader reader = new BinaryDictReader(file); try { - inStream = new FileInputStream(file); - final byte[] buffer = new byte[(int)file.length()]; - inStream.read(buffer); - - UserHistoryDictIOUtils.readDictionaryBinary(new ByteArrayWrapper(buffer), listener); + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); } catch (FileNotFoundException e) { Log.e(TAG, "file not found", e); } catch (IOException e) { Log.e(TAG, "IOException", e); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } } + UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); } public void testGenerateFusionDictionary() { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index c2c77d61a..d0b460af0 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; @@ -185,16 +186,15 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final FileInputStream inStream = new FileInputStream(decodedSpec.mFile); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length()); + final BinaryDictReader reader = new BinaryDictReader(decodedSpec.mFile); + reader.openBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); + return BinaryDictInputOutput.readDictionaryBinary(reader, null); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index cc890f60c..9bce988ac 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.MakedictLog; @@ -265,24 +266,10 @@ public class DictionaryMaker { */ private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { - FileInputStream inStream = null; - - try { - final File file = new File(binaryFilename); - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } + final File file = new File(binaryFilename); + final BinaryDictReader reader = new BinaryDictReader(file); + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + return BinaryDictInputOutput.readDictionaryBinary(reader, null); } /** diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 554bd2478..fb1cc8493 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; @@ -69,11 +70,9 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final FileInputStream inStream = new FileInputStream(decodeSpec.mFile); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, decodeSpec.mFile.length()); - final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), + final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile); + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader, null /* dict : an optional dictionary to add words to, or null */); assertEquals("Dictionary can't be read back correctly", resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);