From 9c13d52d2372b872876c9926e366075fdf35a606 Mon Sep 17 00:00:00 2001 From: Yuichiro Hanada Date: Wed, 16 Oct 2013 16:45:24 +0900 Subject: [PATCH] [Refactor] Add BinaryDictUtils. Change-Id: I95c5fb58e2cab3a2c523d2bc3c6fbad06692ac59 --- .../BinaryDictDecoderEncoderTests.java | 232 ++++++++---------- .../makedict/BinaryDictIOUtilsTests.java | 130 ++++------ .../latin/makedict/BinaryDictUtils.java | 86 +++++++ 3 files changed, 235 insertions(+), 213 deletions(-) create mode 100644 tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 32c07e106..d670aad43 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -26,7 +26,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -60,9 +59,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; private static final int NUM_OF_SHORTCUTS = 5; - private static final int USE_BYTE_ARRAY = 1; - private static final int USE_BYTE_BUFFER = 2; - private static final ArrayList sWords = CollectionUtils.newArrayList(); private static final SparseArray> sEmptyBigrams = CollectionUtils.newSparseArray(); @@ -71,21 +67,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { CollectionUtils.newSparseArray(); private static final HashMap> sShortcuts = CollectionUtils.newHashMap(); - private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); - private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */, - true /* hasTimestamp */); - - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - public BinaryDictDecoderEncoderTests() { this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); } @@ -124,17 +105,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - return new Ver4DictEncoder(getContext().getCacheDir()); - } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { - return new Ver3DictEncoder(file); - } else { - throw new RuntimeException("The format option has a wrong version : " - + formatOptions.mVersion); - } - } - private void generateWords(final int number, final Random random, final int[] codePointSet) { final Set wordSet = CollectionUtils.newHashSet(); while (wordSet.size() < number) { @@ -186,7 +156,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictEncoder dictEncoder = getDictEncoder(file, formatOptions); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); now = System.currentTimeMillis(); // If you need to dump the dict to a textual file, uncomment the line below and the @@ -241,54 +212,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String outputOptions(final int bufferType, final FormatSpec.FormatOptions formatOptions) { String result = " : buffer type = " - + ((bufferType == USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); + + ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); result += " : version = " + formatOptions.mVersion; return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; } - private DictionaryOptions getDictionaryOptions(final String id, final String version) { - final DictionaryOptions options = new DictionaryOptions(new HashMap(), - false, false); - options.mAttributes.put("version", version); - options.mAttributes.put("dictionary", id); - return options; - } - - private File setUpDictionaryFile(final String name, final String version) { - File file = null; - try { - file = new File(getContext().getCacheDir(), name + "." + version - + TEST_DICT_FILE_EXTENSION); - file.createNewFile(); - } catch (IOException e) { - // do nothing - } - assertTrue("Failed to create the dictionary file.", file.exists()); - return file; - } - - private DictDecoder getDictDecoder(final File file, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - final FileHeader header = new FileHeader(0, dictOptions, formatOptions); - return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(), - header.getId() + "." + header.getVersion()), bufferType); - } else { - return FormatSpec.getDictDecoder(file, bufferType); - } - } // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List words, final SparseArray> bigrams, - final HashMap> shortcutMap, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + final HashMap> shortcutMap, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; @@ -310,17 +248,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final String dictName = "runReadAndWrite"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType, - formatOptions, dict.mOptions); + final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -349,8 +287,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final byte[] buffer = new byte[50 * 3]; final DictBuffer dictBuffer = new ByteArrayDictBuffer(buffer); for (final String word : sWords) { - Log.d("testReadAndWriteString", "write : " + word); - Arrays.fill(buffer, (byte)0); + Arrays.fill(buffer, (byte) 0); CharEncoding.writeString(buffer, 0, word); dictBuffer.position(0); final String str = CharEncoding.readString(dictBuffer); @@ -361,12 +298,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadAndWriteWithByteBuffer() { final List results = CollectionUtils.newArrayList(); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -376,12 +319,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadAndWriteWithByteArray() { final List results = CollectionUtils.newArrayList(); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -437,8 +386,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List words, - final SparseArray> bigrams, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + final SparseArray> bigrams, final int bufferType) { FileInputStream inStream = null; final TreeMap resultWords = CollectionUtils.newTreeMap(); @@ -448,8 +396,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -476,20 +423,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions, final String message) { final String dictName = "runReadUnigrams"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, - formatOptions, dict.mOptions); + long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType); long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, - bufferType, formatOptions, dict.mOptions); + bufferType); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap + " : " + message + " : " + outputOptions(bufferType, formatOptions); @@ -508,13 +455,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadUnigramsAndBigramsBinaryWithByteBuffer() { final ArrayList results = CollectionUtils.newArrayList(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -524,13 +476,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadUnigramsAndBigramsBinaryWithByteArray() { final ArrayList results = CollectionUtils.newArrayList(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -578,16 +535,16 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatOptions formatOptions, final String message) { final String dictName = "testGetTerminalPosition"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY, - formatOptions, dict.mOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { @@ -638,19 +595,29 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testGetTerminalPosition() { final ArrayList results = CollectionUtils.newArrayList(); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -660,7 +627,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private void runTestDeleteWord(final FormatOptions formatOptions) { final String dictName = "testDeleteWord"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( @@ -668,15 +636,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, formatOptions); - final DictUpdater dictUpdater; - if (formatOptions.mVersion == 3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else if (formatOptions.mVersion == 4) { - dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion - + " doesn't exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, @@ -696,7 +656,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testDeleteWord() { - runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE); - runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index afe5adb73..592f86e94 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -23,6 +23,7 @@ import android.util.Log; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -30,24 +31,16 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Random; @LargeTest public class BinaryDictIOUtilsTests extends AndroidTestCase { private static final String TAG = BinaryDictIOUtilsTests.class.getSimpleName(); - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3, true); private static final ArrayList sWords = CollectionUtils.newArrayList(); public static final int DEFAULT_MAX_UNIGRAMS = 1500; private final int mMaxUnigrams; - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - - private static final int VERSION3 = 3; - private static final int VERSION4 = 4; - private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -141,7 +134,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int position = FormatSpec.NOT_VALID_WORD; try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER); position = dictDecoder.getTerminalPosition(word); } catch (IOException e) { @@ -159,7 +152,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { * @throws IOException * @throws UnsupportedFormatException */ - private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, + private static PtNodeInfo findWordByDictDecoder(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = dictDecoder.getTerminalPosition(word); if (position != FormatSpec.NOT_VALID_WORD) { @@ -176,7 +169,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { PtNodeInfo info = null; try { dictDecoder.openDictBuffer(); - info = findWordByBinaryDictReader(dictDecoder, word); + info = findWordByDictDecoder(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { } @@ -186,16 +179,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { // return amount of time to insert a word private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList bigrams, - final ArrayList shortcuts, final int formatVersion) { + final ArrayList shortcuts, final FormatOptions formatOptions) { long amountOfTime = -1; try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); @@ -212,18 +199,14 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { return amountOfTime; } - private void deleteWord(final File file, final String word, final int formatVersion) { + private void deleteWord(final File file, final String word, final FormatOptions formatOptions) { try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); dictUpdater.deleteWord(word); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while deleting a word", e); } catch (UnsupportedFormatException e) { + Log.e(TAG, "Raised an UnsupportedFormatException while deleting a word", e); } } @@ -242,23 +225,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - private void runTestInsertWord(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } + private void runTestInsertWord(final FormatOptions formatOptions) { + final String testName = "testInsertWord"; + final String version = Long.toString(System.currentTimeMillis()); + final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap(), false, false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("abcd", 10, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -266,54 +247,52 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcde", 10, false, null, null, formatOptions); - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatOptions); checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatOptions); checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion); + insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatOptions); // split 1 - insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion); + insertAndCheckWord(file, "ab", 20, false, null, null, formatOptions); // split 2 - insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion); + insertAndCheckWord(file, "ami", 30, false, null, null, formatOptions); - deleteWord(file, "ami", formatVersion); + deleteWord(file, "ami", formatOptions); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatOptions); - deleteWord(file, "abcd", formatVersion); + deleteWord(file, "abcd", formatOptions); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); } public void testInsertWord() { - runTestInsertWord(VERSION3); + runTestInsertWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } - private void runTestInsertWordWithBigrams(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } + private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) { + final String testName = "testInsertWordWithBigrams"; + final String version = Long.toString(System.currentTimeMillis()); + File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap(), false, false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("abcd", 10, null, false); dict.add("efgh", 15, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -323,8 +302,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { final ArrayList banana = new ArrayList(); banana.add(new WeightedString("banana", 10)); - insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion); - insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion); + insertAndCheckWord(file, "banana", 0, false, null, null, formatOptions); + insertAndCheckWord(file, "recursive", 60, true, banana, null, formatOptions); final PtNodeInfo info = findWordFromFile(file, "recursive"); int bananaPos = getWordPosition(file, "banana"); @@ -334,27 +313,24 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(VERSION3); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } - private void runTestRandomWords(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - } - assertNotNull(file); + private void runTestRandomWords(final FormatOptions formatOptions) { + final String testName = "testRandomWord"; + final String version = Long.toString(System.currentTimeMillis()); + final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap(), false, - false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("initial", 10, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { assertTrue(false); } catch (UnsupportedFormatException e) { @@ -366,7 +342,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int cnt = 0; for (final String word : sWords) { final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion); + cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatOptions); maxTimeToInsert = Math.max(maxTimeToInsert, diff); minTimeToInsert = Math.min(minTimeToInsert, diff); sum += diff; @@ -377,13 +353,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } - Log.d(TAG, "Test version " + formatVersion); + Log.d(TAG, "Test version " + formatOptions); Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); } public void testRandomWords() { - runTestRandomWords(VERSION3); + runTestRandomWords(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java new file mode 100644 index 000000000..f476738f3 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; + +import java.io.File; +import java.util.HashMap; + +public class BinaryDictUtils { + public static final int USE_BYTE_ARRAY = 1; + public static final int USE_BYTE_BUFFER = 2; + + public static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + + public static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); + public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */, + true /* hasTimestamp */); + + public static DictionaryOptions getDictionaryOptions(final String id, final String version) { + final DictionaryOptions options = new DictionaryOptions(new HashMap(), + false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */); + options.mAttributes.put("dictionary", id); + options.mAttributes.put("version", version); + return options; + } + + public static File getDictFile(final String name, final String version, + final FormatOptions formatOptions, final File directory) { + if (formatOptions.mVersion == 2 || formatOptions.mVersion == 3) { + return new File(directory, name + "." + version + TEST_DICT_FILE_EXTENSION); + } else if (formatOptions.mVersion == 4) { + return new File(directory, name + "." + version); + } else { + throw new RuntimeException("the format option has a wrong version : " + + formatOptions.mVersion); + } + } + + public static DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions, + final File cacheDir) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictEncoder(cacheDir); + } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { + return new Ver3DictEncoder(file); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } + + public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else if (formatOptions.mVersion == 3) { + return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } +}