/* * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.text.TextUtils; import android.util.Pair; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.Random; // TODO Use the seed passed as an argument for makedict test. @LargeTest public class BinaryDictionaryTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; @Override protected void setUp() throws Exception { super.setUp(); } @Override protected void tearDown() throws Exception { super.tearDown(); } private File createEmptyDictionaryAndGetFile(final String dictId, final int formatVersion) throws IOException { if (formatVersion == FormatSpec.VERSION4) { return createEmptyVer4DictionaryAndGetFile(dictId); } else { throw new IOException("Dictionary format version " + formatVersion + " is not supported."); } } private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException { final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); file.delete(); file.mkdir(); Map attributeMap = new HashMap(); if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, Locale.ENGLISH, attributeMap)) { return file; } else { throw new IOException("Empty dictionary " + file.getAbsolutePath() + " cannot be created."); } } public void testIsValidDictionary() { testIsValidDictionary(FormatSpec.VERSION4); } private void testIsValidDictionary(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); assertTrue("binaryDictionary must be valid for existing valid dictionary file.", binaryDictionary.isValidDictionary()); binaryDictionary.close(); assertFalse("binaryDictionary must be invalid after closing.", binaryDictionary.isValidDictionary()); FileUtils.deleteRecursively(dictFile); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); assertFalse("binaryDictionary must be invalid for not existing dictionary file.", binaryDictionary.isValidDictionary()); binaryDictionary.close(); } public void testAddTooLongWord() { testAddTooLongWord(FormatSpec.VERSION4); } private void testAddTooLongWord(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final StringBuffer stringBuilder = new StringBuffer(); for (int i = 0; i < Constants.DICTIONARY_MAX_WORD_LENGTH; i++) { stringBuilder.append('a'); } final String validLongWord = stringBuilder.toString(); stringBuilder.append('a'); final String invalidLongWord = stringBuilder.toString(); final int probability = 100; addUnigramWord(binaryDictionary, "aaa", probability); addUnigramWord(binaryDictionary, validLongWord, probability); addUnigramWord(binaryDictionary, invalidLongWord, probability); // Too long short cut. binaryDictionary.addUnigramWord("a", probability, invalidLongWord, 10 /* shortcutProbability */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); addUnigramWord(binaryDictionary, "abc", probability); final int updatedProbability = 200; // Update. addUnigramWord(binaryDictionary, validLongWord, updatedProbability); addUnigramWord(binaryDictionary, invalidLongWord, updatedProbability); addUnigramWord(binaryDictionary, "abc", updatedProbability); assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(updatedProbability, binaryDictionary.getFrequency(validLongWord)); assertEquals(BinaryDictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency(invalidLongWord)); assertEquals(updatedProbability, binaryDictionary.getFrequency("abc")); dictFile.delete(); } private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); } private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, final String word1, final int probability) { binaryDictionary.addBigramWords(word0, word1, probability, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); } public void testAddUnigramWord() { testAddUnigramWord(FormatSpec.VERSION4); } private void testAddUnigramWord(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; addUnigramWord(binaryDictionary, "aaa", probability); // Reallocate and create. addUnigramWord(binaryDictionary, "aab", probability); // Insert into children. addUnigramWord(binaryDictionary, "aac", probability); // Make terminal. addUnigramWord(binaryDictionary, "aa", probability); // Create children. addUnigramWord(binaryDictionary, "aaaa", probability); // Reallocate and make termianl. addUnigramWord(binaryDictionary, "a", probability); final int updatedProbability = 200; // Update. addUnigramWord(binaryDictionary, "aaa", updatedProbability); assertEquals(probability, binaryDictionary.getFrequency("aab")); assertEquals(probability, binaryDictionary.getFrequency("aac")); assertEquals(probability, binaryDictionary.getFrequency("aa")); assertEquals(probability, binaryDictionary.getFrequency("aaaa")); assertEquals(probability, binaryDictionary.getFrequency("a")); assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa")); dictFile.delete(); } public void testRandomlyAddUnigramWord() { testRandomlyAddUnigramWord(FormatSpec.VERSION4); } private void testRandomlyAddUnigramWord(final int formatVersion) { final int wordCount = 1000; final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final HashMap probabilityMap = new HashMap(); // Test a word that isn't contained within the dictionary. final Random random = new Random(seed); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); probabilityMap.put(word, random.nextInt(0xFF)); } for (String word : probabilityMap.keySet()) { addUnigramWord(binaryDictionary, word, probabilityMap.get(word)); } for (String word : probabilityMap.keySet()) { assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); } dictFile.delete(); } public void testAddBigramWords() { testAddBigramWords(FormatSpec.VERSION4); } private void testAddBigramWords(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; final int bigramProbability = 10; final int updatedBigramProbability = 15; addUnigramWord(binaryDictionary, "aaa", unigramProbability); addUnigramWord(binaryDictionary, "abb", unigramProbability); addUnigramWord(binaryDictionary, "bcc", unigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); final int probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability, updatedBigramProbability); assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getBigramProbability("bcc", "aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getBigramProbability("bcc", "bbc")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getBigramProbability("aaa", "aaa")); // Testing bigram link. addUnigramWord(binaryDictionary, "abcde", unigramProbability); addUnigramWord(binaryDictionary, "fghij", unigramProbability); addBigramWords(binaryDictionary, "abcde", "fghij", bigramProbability); addUnigramWord(binaryDictionary, "fgh", unigramProbability); addUnigramWord(binaryDictionary, "abc", unigramProbability); addUnigramWord(binaryDictionary, "f", unigramProbability); assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getBigramProbability("abcde", "fgh")); addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij")); dictFile.delete(); } public void testRandomlyAddBigramWords() { testRandomlyAddBigramWords(FormatSpec.VERSION4); } private void testRandomlyAddBigramWords(final int formatVersion) { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final ArrayList words = new ArrayList(); final ArrayList> bigramWords = new ArrayList>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final HashMap unigramProbabilities = new HashMap(); final HashMap, Integer> bigramProbabilities = new HashMap, Integer>(); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { final String word0 = words.get(random.nextInt(wordCount)); final String word1 = words.get(random.nextInt(wordCount)); if (TextUtils.equals(word0, word1)) { continue; } final Pair bigram = new Pair(word0, word1); bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); bigramProbabilities.put(bigram, bigramProbability); addBigramWords(binaryDictionary, word0, word1, bigramProbability); } for (final Pair bigram : bigramWords) { final int unigramProbability = unigramProbabilities.get(bigram.second); final int bigramProbability = bigramProbabilities.get(bigram); final int probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); assertEquals(probability, binaryDictionary.getBigramProbability(bigram.first, bigram.second)); } dictFile.delete(); } public void testRemoveBigramWords() { testRemoveBigramWords(FormatSpec.VERSION4); } private void testRemoveBigramWords(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; final int bigramProbability = 10; addUnigramWord(binaryDictionary, "aaa", unigramProbability); addUnigramWord(binaryDictionary, "abb", unigramProbability); addUnigramWord(binaryDictionary, "bcc", unigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); binaryDictionary.removeBigramWords("aaa", "abb"); assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb")); addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); binaryDictionary.removeBigramWords("aaa", "bcc"); assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc")); binaryDictionary.removeBigramWords("abb", "aaa"); assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa")); binaryDictionary.removeBigramWords("abb", "bcc"); assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc")); binaryDictionary.removeBigramWords("aaa", "abb"); // Test remove non-existing bigram operation. binaryDictionary.removeBigramWords("aaa", "abb"); binaryDictionary.removeBigramWords("bcc", "aaa"); dictFile.delete(); } public void testFlushDictionary() { testFlushDictionary(FormatSpec.VERSION4); } private void testFlushDictionary(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; addUnigramWord(binaryDictionary, "aaa", probability); addUnigramWord(binaryDictionary, "abcd", probability); // Close without flushing. binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd")); addUnigramWord(binaryDictionary, "aaa", probability); addUnigramWord(binaryDictionary, "abcd", probability); binaryDictionary.flush(); binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(probability, binaryDictionary.getFrequency("abcd")); addUnigramWord(binaryDictionary, "bcde", probability); binaryDictionary.flush(); binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); assertEquals(probability, binaryDictionary.getFrequency("bcde")); binaryDictionary.close(); dictFile.delete(); } public void testFlushWithGCDictionary() { testFlushWithGCDictionary(FormatSpec.VERSION4); } private void testFlushWithGCDictionary(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; final int bigramProbability = 10; addUnigramWord(binaryDictionary, "aaa", unigramProbability); addUnigramWord(binaryDictionary, "abb", unigramProbability); addUnigramWord(binaryDictionary, "bcc", unigramProbability); addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); binaryDictionary.flushWithGC(); binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); binaryDictionary.flushWithGC(); binaryDictionary.close(); dictFile.delete(); } public void testAddBigramWordsAndFlashWithGC() { testAddBigramWordsAndFlashWithGC(FormatSpec.VERSION4); } // TODO: Evaluate performance of GC private void testAddBigramWordsAndFlashWithGC(final int formatVersion) { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 30; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final ArrayList words = new ArrayList(); final ArrayList> bigramWords = new ArrayList>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final HashMap unigramProbabilities = new HashMap(); final HashMap, Integer> bigramProbabilities = new HashMap, Integer>(); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { final String word0 = words.get(random.nextInt(wordCount)); final String word1 = words.get(random.nextInt(wordCount)); if (TextUtils.equals(word0, word1)) { continue; } final Pair bigram = new Pair(word0, word1); bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); bigramProbabilities.put(bigram, bigramProbability); addBigramWords(binaryDictionary, word0, word1, bigramProbability); } binaryDictionary.flushWithGC(); binaryDictionary.close(); binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); for (final Pair bigram : bigramWords) { final int unigramProbability = unigramProbabilities.get(bigram.second); final int bigramProbability = bigramProbabilities.get(bigram); final int probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); assertEquals(probability, binaryDictionary.getBigramProbability(bigram.first, bigram.second)); } dictFile.delete(); } public void testRandomOperationsAndFlashWithGC() { testRandomOperationsAndFlashWithGC(FormatSpec.VERSION4); } private void testRandomOperationsAndFlashWithGC(final int formatVersion) { final int flashWithGCIterationCount = 50; final int operationCountInEachIteration = 200; final int initialUnigramCount = 100; final float addUnigramProb = 0.5f; final float addBigramProb = 0.8f; final float removeBigramProb = 0.2f; final int codePointSetSize = 30; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final ArrayList words = new ArrayList(); final ArrayList> bigramWords = new ArrayList>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final HashMap unigramProbabilities = new HashMap(); final HashMap, Integer> bigramProbabilities = new HashMap, Integer>(); for (int i = 0; i < initialUnigramCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); addUnigramWord(binaryDictionary, word, unigramProbability); } binaryDictionary.flushWithGC(); binaryDictionary.close(); for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) { binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) { // Add unigram. if (random.nextFloat() < addUnigramProb) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); addUnigramWord(binaryDictionary, word, unigramProbability); } // Add bigram. if (random.nextFloat() < addBigramProb && words.size() > 2) { final int word0Index = random.nextInt(words.size()); int word1Index = random.nextInt(words.size() - 1); if (word0Index <= word1Index) { word1Index++; } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); if (TextUtils.equals(word0, word1)) { continue; } final int bigramProbability = random.nextInt(0xF); final Pair bigram = new Pair(word0, word1); bigramWords.add(bigram); bigramProbabilities.put(bigram, bigramProbability); addBigramWords(binaryDictionary, word0, word1, bigramProbability); } // Remove bigram. if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { final int bigramIndex = random.nextInt(bigramWords.size()); final Pair bigram = bigramWords.get(bigramIndex); bigramWords.remove(bigramIndex); bigramProbabilities.remove(bigram); binaryDictionary.removeBigramWords(bigram.first, bigram.second); } } // Test whether the all unigram operations are collectlly handled. for (int i = 0; i < words.size(); i++) { final String word = words.get(i); final int unigramProbability = unigramProbabilities.get(word); assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); } // Test whether the all bigram operations are collectlly handled. for (int i = 0; i < bigramWords.size(); i++) { final Pair bigram = bigramWords.get(i); final int unigramProbability = unigramProbabilities.get(bigram.second); final int probability; if (bigramProbabilities.containsKey(bigram)) { final int bigramProbability = bigramProbabilities.get(bigram); probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); } else { probability = Dictionary.NOT_A_PROBABILITY; } assertEquals(probability, binaryDictionary.getBigramProbability(bigram.first, bigram.second)); } binaryDictionary.flushWithGC(); binaryDictionary.close(); } dictFile.delete(); } public void testAddManyUnigramsAndFlushWithGC() { testAddManyUnigramsAndFlushWithGC(FormatSpec.VERSION4); } private void testAddManyUnigramsAndFlushWithGC(final int formatVersion) { final int flashWithGCIterationCount = 3; final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final ArrayList words = new ArrayList(); final HashMap unigramProbabilities = new HashMap(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); BinaryDictionary binaryDictionary; for (int i = 0; i < flashWithGCIterationCount; i++) { binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < words.size(); j++) { final String word = words.get(j); final int unigramProbability = unigramProbabilities.get(word); assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); } binaryDictionary.flushWithGC(); binaryDictionary.close(); } dictFile.delete(); } public void testUnigramAndBigramCount() { testUnigramAndBigramCount(FormatSpec.VERSION4); } private void testUnigramAndBigramCount(final int formatVersion) { final int flashWithGCIterationCount = 10; final int codePointSetSize = 50; final int unigramCountPerIteration = 1000; final int bigramCountPerIteration = 2000; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final ArrayList words = new ArrayList(); final HashSet> bigrams = new HashSet>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); BinaryDictionary binaryDictionary; for (int i = 0; i < flashWithGCIterationCount; i++) { binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); for (int j = 0; j < unigramCountPerIteration; j++) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < bigramCountPerIteration; j++) { final String word0 = words.get(random.nextInt(words.size())); final String word1 = words.get(random.nextInt(words.size())); if (TextUtils.equals(word0, word1)) { continue; } bigrams.add(new Pair(word0, word1)); final int bigramProbability = random.nextInt(0xF); addBigramWords(binaryDictionary, word0, word1, bigramProbability); } assertEquals(new HashSet(words).size(), Integer.parseInt( binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); assertEquals(new HashSet>(bigrams).size(), Integer.parseInt( binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.flushWithGC(); assertEquals(new HashSet(words).size(), Integer.parseInt( binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); assertEquals(new HashSet>(bigrams).size(), Integer.parseInt( binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.close(); } dictFile.delete(); } public void testAddMultipleDictionaryEntries() { testAddMultipleDictionaryEntries(FormatSpec.VERSION4); } private void testAddMultipleDictionaryEntries(final int formatVersion) { final int codePointSetSize = 20; final int lmParamCount = 1000; final double bigramContinueRate = 0.9; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final HashMap unigramProbabilities = new HashMap(); final HashMap, Integer> bigramProbabilities = new HashMap, Integer>(); final LanguageModelParam[] languageModelParams = new LanguageModelParam[lmParamCount]; String prevWord = null; for (int i = 0; i < languageModelParams.length; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final int probability = random.nextInt(0xFF); final int bigramProbability = random.nextInt(0xF); unigramProbabilities.put(word, probability); if (prevWord == null) { languageModelParams[i] = new LanguageModelParam(word, probability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } else { languageModelParams[i] = new LanguageModelParam(prevWord, word, probability, bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); bigramProbabilities.put(new Pair(prevWord, word), bigramProbability); } prevWord = (random.nextDouble() < bigramContinueRate) ? word : null; } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); binaryDictionary.addMultipleDictionaryEntries(languageModelParams); for (Map.Entry entry : unigramProbabilities.entrySet()) { assertEquals((int)entry.getValue(), binaryDictionary.getFrequency(entry.getKey())); } for (Map.Entry, Integer> entry : bigramProbabilities.entrySet()) { final String word0 = entry.getKey().first; final String word1 = entry.getKey().second; final int unigramProbability = unigramProbabilities.get(word1); final int bigramProbability = entry.getValue(); final int probability = binaryDictionary.calculateProbability( unigramProbability, bigramProbability); assertEquals(probability, binaryDictionary.getBigramProbability(word0, word1)); } } public void testGetWordProperties() { testGetWordProperties(FormatSpec.VERSION4); } private void testGetWordProperties(final int formatVersion) { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); final int UNIGRAM_COUNT = 1000; final int BIGRAM_COUNT = 1000; final int codePointSetSize = 20; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord"); assertFalse(invalidWordProperty.isValid()); final ArrayList words = new ArrayList(); final HashMap wordProbabilities = new HashMap(); final HashMap> bigrams = new HashMap>(); final HashMap, Integer> bigramProbabilities = new HashMap, Integer>(); for (int i = 0; i < UNIGRAM_COUNT; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final int unigramProbability = random.nextInt(0xFF); final boolean isNotAWord = random.nextBoolean(); final boolean isBlacklisted = random.nextBoolean(); // TODO: Add tests for historical info. binaryDictionary.addUnigramWord(word, unigramProbability, null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY, isNotAWord, isBlacklisted, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } words.add(word); wordProbabilities.put(word, unigramProbability); final WordProperty wordProperty = binaryDictionary.getWordProperty(word); assertEquals(word, wordProperty.mWord); assertTrue(wordProperty.isValid()); assertEquals(isNotAWord, wordProperty.mIsNotAWord); assertEquals(isBlacklisted, wordProperty.mIsBlacklistEntry); assertEquals(false, wordProperty.mHasBigrams); assertEquals(false, wordProperty.mHasShortcuts); assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability); assertTrue(wordProperty.mShortcutTargets.isEmpty()); } for (int i = 0; i < BIGRAM_COUNT; i++) { final int word0Index = random.nextInt(wordProbabilities.size()); final int word1Index = random.nextInt(wordProbabilities.size()); if (word0Index == word1Index) { continue; } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); final int bigramProbability = random.nextInt(0xF); binaryDictionary.addBigramWords(word0, word1, bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } if (!bigrams.containsKey(word0)) { final HashSet bigramWord1s = new HashSet(); bigrams.put(word0, bigramWord1s); } bigrams.get(word0).add(word1); bigramProbabilities.put(new Pair(word0, word1), bigramProbability); } for (int i = 0; i < words.size(); i++) { final String word0 = words.get(i); if (!bigrams.containsKey(word0)) { continue; } final HashSet bigramWord1s = bigrams.get(word0); final WordProperty wordProperty = binaryDictionary.getWordProperty(word0); assertEquals(bigramWord1s.size(), wordProperty.mBigrams.size()); for (int j = 0; j < wordProperty.mBigrams.size(); j++) { final String word1 = wordProperty.mBigrams.get(j).mWord; assertTrue(bigramWord1s.contains(word1)); final int probability = wordProperty.mBigrams.get(j).getProbability(); assertEquals((int)bigramProbabilities.get(new Pair(word0, word1)), probability); assertEquals(wordProperty.mBigrams.get(j).getProbability(), probability); } } } public void testIterateAllWords() { testIterateAllWords(FormatSpec.VERSION4); } private void testIterateAllWords(final int formatVersion) { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); final int UNIGRAM_COUNT = 1000; final int BIGRAM_COUNT = 1000; final int codePointSetSize = 20; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord"); assertFalse(invalidWordProperty.isValid()); final ArrayList words = new ArrayList(); final HashMap wordProbabilitiesToCheckLater = new HashMap(); final HashMap> bigrams = new HashMap>(); final HashMap, Integer> bigramProbabilitiesToCheckLater = new HashMap, Integer>(); for (int i = 0; i < UNIGRAM_COUNT; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final int unigramProbability = random.nextInt(0xFF); addUnigramWord(binaryDictionary, word, unigramProbability); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } words.add(word); wordProbabilitiesToCheckLater.put(word, unigramProbability); } for (int i = 0; i < BIGRAM_COUNT; i++) { final int word0Index = random.nextInt(wordProbabilitiesToCheckLater.size()); final int word1Index = random.nextInt(wordProbabilitiesToCheckLater.size()); if (word0Index == word1Index) { continue; } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); final int bigramProbability = random.nextInt(0xF); binaryDictionary.addBigramWords(word0, word1, bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } if (!bigrams.containsKey(word0)) { final HashSet bigramWord1s = new HashSet(); bigrams.put(word0, bigramWord1s); } bigrams.get(word0).add(word1); bigramProbabilitiesToCheckLater.put( new Pair(word0, word1), bigramProbability); } final HashSet wordSet = new HashSet(words); final HashSet> bigramSet = new HashSet>(bigramProbabilitiesToCheckLater.keySet()); int token = 0; do { final BinaryDictionary.GetNextWordPropertyResult result = binaryDictionary.getNextWordProperty(token); final WordProperty wordProperty = result.mWordProperty; final String word0 = wordProperty.mWord; assertEquals((int)wordProbabilitiesToCheckLater.get(word0), wordProperty.mProbabilityInfo.mProbability); wordSet.remove(word0); final HashSet bigramWord1s = bigrams.get(word0); for (int j = 0; j < wordProperty.mBigrams.size(); j++) { final String word1 = wordProperty.mBigrams.get(j).mWord; assertTrue(bigramWord1s.contains(word1)); final int probability = wordProperty.mBigrams.get(j).getProbability(); final Pair bigram = new Pair(word0, word1); assertEquals((int)bigramProbabilitiesToCheckLater.get(bigram), probability); bigramSet.remove(bigram); } token = result.mNextToken; } while (token != 0); assertTrue(wordSet.isEmpty()); assertTrue(bigramSet.isEmpty()); } public void testAddShortcuts() { testAddShortcuts(FormatSpec.VERSION4); } private void testAddShortcuts(final int formatVersion) { File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; final int shortcutProbability = 10; binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz", shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); WordProperty wordProperty = binaryDictionary.getWordProperty("aaa"); assertEquals(1, wordProperty.mShortcutTargets.size()); assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); final int updatedShortcutProbability = 2; binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz", updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); wordProperty = binaryDictionary.getWordProperty("aaa"); assertEquals(1, wordProperty.mShortcutTargets.size()); assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); assertEquals(updatedShortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy", shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); final HashMap shortcutTargets = new HashMap(); shortcutTargets.put("zzz", updatedShortcutProbability); shortcutTargets.put("yyy", shortcutProbability); wordProperty = binaryDictionary.getWordProperty("aaa"); assertEquals(2, wordProperty.mShortcutTargets.size()); for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.getProbability()); shortcutTargets.remove(shortcutTarget.mWord); } shortcutTargets.put("zzz", updatedShortcutProbability); shortcutTargets.put("yyy", shortcutProbability); binaryDictionary.flushWithGC(); wordProperty = binaryDictionary.getWordProperty("aaa"); assertEquals(2, wordProperty.mShortcutTargets.size()); for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.getProbability()); shortcutTargets.remove(shortcutTarget.mWord); } } public void testAddManyShortcuts() { testAddManyShortcuts(FormatSpec.VERSION4); } private void testAddManyShortcuts(final int formatVersion) { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); final int UNIGRAM_COUNT = 1000; final int SHORTCUT_COUNT = 10000; final int codePointSetSize = 20; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final ArrayList words = new ArrayList(); final HashMap unigramProbabilities = new HashMap(); final HashMap> shortcutTargets = new HashMap>(); File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); for (int i = 0; i < UNIGRAM_COUNT; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final int unigramProbability = random.nextInt(0xFF); addUnigramWord(binaryDictionary, word, unigramProbability); words.add(word); unigramProbabilities.put(word, unigramProbability); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } } for (int i = 0; i < SHORTCUT_COUNT; i++) { final String shortcutTarget = CodePointUtils.generateWord(random, codePointSet); final int shortcutProbability = random.nextInt(0xF); final String word = words.get(random.nextInt(words.size())); final int unigramProbability = unigramProbabilities.get(word); binaryDictionary.addUnigramWord(word, unigramProbability, shortcutTarget, shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); if (shortcutTargets.containsKey(word)) { final HashMap shortcutTargetsOfWord = shortcutTargets.get(word); shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); } else { final HashMap shortcutTargetsOfWord = new HashMap(); shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); shortcutTargets.put(word, shortcutTargetsOfWord); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); } } for (final String word : words) { final WordProperty wordProperty = binaryDictionary.getWordProperty(word); assertEquals((int)unigramProbabilities.get(word), wordProperty.mProbabilityInfo.mProbability); if (!shortcutTargets.containsKey(word)) { // The word does not have shortcut targets. continue; } assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size()); for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { final String targetCodePonts = shortcutTarget.mWord; assertEquals((int)shortcutTargets.get(word).get(targetCodePonts), shortcutTarget.getProbability()); } } } }