nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Consolidate WordProperty and Word. 

Bug: 11281877
Bug: 12810574
Change-Id: I9dc99188f80f25a8780c1860dab46e4aa80a23e5
main
Keisuke Kuroyanagi 2014-02-06 15:13:33 +09:00
parent df1d3e733e
commit 5f5feeba13
16 changed files with 341 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -26,12 +26,12 @@ import com.android.inputmethod.latin.makedict.DictionaryHeader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.settings.NativeSuggestOptions;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.JniUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.StringUtils;
import com.android.inputmethod.latin.utils.WordProperty;
import java.io.File;
import java.util.ArrayList;

2
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -23,13 +23,13 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.makedict.DictionaryHeader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
import com.android.inputmethod.latin.utils.WordProperty;
import java.io.File;
import java.util.ArrayList;

16
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -606,19 +606,21 @@ public final class BinaryDictDecoderUtils {
FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions);
if (null != dict) {
for (final Word w : dict) {
if (w.mIsBlacklistEntry) {
newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
for (final WordProperty wordProperty : dict) {
if (wordProperty.mIsBlacklistEntry) {
newDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets,
wordProperty.mIsNotAWord);
} else {
newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
newDict.add(wordProperty.mWord, wordProperty.getProbability(),
wordProperty.mShortcutTargets, wordProperty.mIsNotAWord);
}
}
for (final Word w : dict) {
for (final WordProperty wordProperty : dict) {
// By construction a binary dictionary may not have bigrams pointing to
// words that are not also registered as unigrams so we don't have to avoid
// them explicitly here.
for (final WeightedString bigram : w.mBigrams) {
newDict.setBigram(w.mWord, bigram.mWord, bigram.getProbability());
for (final WeightedString bigram : wordProperty.mBigrams) {
newDict.setBigram(wordProperty.mWord, bigram.mWord, bigram.getProbability());
}
}
}

20
java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
View File

@ -31,7 +31,7 @@ import java.util.LinkedList;
* A dictionary that can fusion heads and tails of words for more compression.
*/
@UsedForTesting
public final class FusionDictionary implements Iterable<Word> {
public final class FusionDictionary implements Iterable<WordProperty> {
private static final boolean DBG = MakedictLog.DBG;
private static int CHARACTER_NOT_FOUND_INDEX = -1;
@ -76,8 +76,12 @@ public final class FusionDictionary implements Iterable<Word> {
public ProbabilityInfo mProbabilityInfo;
public WeightedString(final String word, final int probability) {
this(word, new ProbabilityInfo(probability));
}
public WeightedString(final String word, final ProbabilityInfo probabilityInfo) {
mWord = word;
mProbabilityInfo = new ProbabilityInfo(probability);
mProbabilityInfo = probabilityInfo;
}
public int getProbability() {
@ -90,9 +94,7 @@ public final class FusionDictionary implements Iterable<Word> {
@Override
public int hashCode() {
return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo.mProbability,
mProbabilityInfo.mTimestamp, mProbabilityInfo.mLevel,
mProbabilityInfo.mCount });
return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo});
}
@Override
@ -704,7 +706,7 @@ public final class FusionDictionary implements Iterable<Word> {
*
* This is purely for convenience.
*/
public static final class DictionaryIterator implements Iterator<Word> {
public static final class DictionaryIterator implements Iterator<WordProperty> {
private static final class Position {
public Iterator<PtNode> pos;
public int length;
@ -734,7 +736,7 @@ public final class FusionDictionary implements Iterable<Word> {
}
@Override
public Word next() {
public WordProperty next() {
Position currentPos = mPositions.getLast();
mCurrentString.setLength(currentPos.length);
@ -751,7 +753,7 @@ public final class FusionDictionary implements Iterable<Word> {
mPositions.addLast(currentPos);
}
if (currentPtNode.mFrequency >= 0) {
return new Word(mCurrentString.toString(), currentPtNode.mFrequency,
return new WordProperty(mCurrentString.toString(), currentPtNode.mFrequency,
currentPtNode.mShortcutTargets, currentPtNode.mBigrams,
currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry);
}
@ -777,7 +779,7 @@ public final class FusionDictionary implements Iterable<Word> {
* and say : for (Word w : x) {}
*/
@Override
public Iterator<Word> iterator() {
public Iterator<WordProperty> iterator() {
return new DictionaryIterator(mRootNodeArray.mData);
}
}

14
java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java
View File

@ -19,6 +19,8 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.util.Arrays;
public final class ProbabilityInfo {
public final int mProbability;
// mTimestamp, mLevel and mCount are historical info. These values are depend on the
@ -44,9 +46,19 @@ public final class ProbabilityInfo {
return mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP;
}
@Override
public int hashCode() {
if (hasHistoricalInfo()) {
return Arrays.hashCode(new Object[] { mProbability, mTimestamp, mLevel, mCount });
} else {
return Arrays.hashCode(new Object[] { mProbability });
}
}
@Override
public String toString() {
return mTimestamp + ":" + mLevel + ":" + mCount;
return "f=" + mProbability + (hasHistoricalInfo() ?
",historicalInfo=" + mTimestamp + ":" + mLevel + ":" + mCount : "");
}
@Override

24
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -69,27 +69,29 @@ public class Ver4DictEncoder implements DictEncoder {
// Somehow createEmptyDictFile returned true, but the file was not created correctly
throw new IOException("Cannot create dictionary file");
}
for (final Word word : dict) {
for (final WordProperty wordProperty : dict) {
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts
if (null == word.mShortcutTargets || word.mShortcutTargets.isEmpty()) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(),
null /* shortcutTarget */, 0 /* shortcutProbability */,
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */);
} else {
for (final WeightedString shortcutTarget : word.mShortcutTargets) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(),
shortcutTarget.mWord, shortcutTarget.getProbability(),
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry,
0 /* timestamp */);
}
}
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();
}
}
for (final Word word0 : dict) {
if (null == word0.mBigrams) continue;
for (final WeightedString word1 : word0.mBigrams) {
binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.getProbability(),
for (final WordProperty word0Property : dict) {
if (null == word0Property.mBigrams) continue;
for (final WeightedString word1 : word0Property.mBigrams) {
binaryDict.addBigramWords(word0Property.mWord, word1.mWord, word1.getProbability(),
0 /* timestamp */);
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();

100
java/src/com/android/inputmethod/latin/makedict/Word.java
View File

@ -1,100 +0,0 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Utility class for a word with a frequency.
*
* This is chiefly used to iterate a dictionary.
*/
public final class Word implements Comparable<Word> {
public final String mWord;
public final int mFrequency;
public final ArrayList<WeightedString> mShortcutTargets;
public final ArrayList<WeightedString> mBigrams;
public final boolean mIsNotAWord;
public final boolean mIsBlacklistEntry;
private int mHashCode = 0;
public Word(final String word, final int frequency,
final ArrayList<WeightedString> shortcutTargets,
final ArrayList<WeightedString> bigrams,
final boolean isNotAWord, final boolean isBlacklistEntry) {
mWord = word;
mFrequency = frequency;
mShortcutTargets = shortcutTargets;
mBigrams = bigrams;
mIsNotAWord = isNotAWord;
mIsBlacklistEntry = isBlacklistEntry;
}
private static int computeHashCode(Word word) {
return Arrays.hashCode(new Object[] {
word.mWord,
word.mFrequency,
word.mShortcutTargets.hashCode(),
word.mBigrams.hashCode(),
word.mIsNotAWord,
word.mIsBlacklistEntry
});
}
/**
* Three-way comparison.
*
* A Word x is greater than a word y if x has a higher frequency. If they have the same
* frequency, they are sorted in lexicographic order.
*/
@Override
public int compareTo(Word w) {
if (mFrequency < w.mFrequency) return 1;
if (mFrequency > w.mFrequency) return -1;
return mWord.compareTo(w.mWord);
}
/**
* Equality test.
*
* Words are equal if they have the same frequency, the same spellings, and the same
* attributes.
*/
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (!(o instanceof Word)) return false;
Word w = (Word)o;
return mFrequency == w.mFrequency && mWord.equals(w.mWord)
&& mShortcutTargets.equals(w.mShortcutTargets)
&& mBigrams.equals(w.mBigrams)
&& mIsNotAWord == w.mIsNotAWord
&& mIsBlacklistEntry == w.mIsBlacklistEntry;
}
@Override
public int hashCode() {
if (mHashCode == 0) {
mHashCode = computeHashCode(this);
}
return mHashCode;
}
}

189
java/src/com/android/inputmethod/latin/makedict/WordProperty.java Normal file
View File

@ -0,0 +1,189 @@
/*
* Copyright (C) 2011 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Utility class for a word with a probability.
*
* This is chiefly used to iterate a dictionary.
*/
public final class WordProperty implements Comparable<WordProperty> {
public final String mWord;
public final ProbabilityInfo mProbabilityInfo;
public final ArrayList<WeightedString> mShortcutTargets;
public final ArrayList<WeightedString> mBigrams;
public final boolean mIsNotAWord;
public final boolean mIsBlacklistEntry;
public final boolean mHasShortcuts;
public final boolean mHasBigrams;
private int mHashCode = 0;
public WordProperty(final String word, final int probability,
final ArrayList<WeightedString> shortcutTargets,
final ArrayList<WeightedString> bigrams,
final boolean isNotAWord, final boolean isBlacklistEntry) {
mWord = word;
mProbabilityInfo = new ProbabilityInfo(probability);
mShortcutTargets = shortcutTargets;
mBigrams = bigrams;
mIsNotAWord = isNotAWord;
mIsBlacklistEntry = isBlacklistEntry;
mHasBigrams = !bigrams.isEmpty();
mHasShortcuts = !shortcutTargets.isEmpty();
}
private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) {
return new ProbabilityInfo(
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX]);
}
// Construct word property using information from native code.
// This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY.
public WordProperty(final int[] codePoints, final boolean isNotAWord,
final boolean isBlacklisted, final boolean hasBigram,
final boolean hasShortcuts, final int[] probabilityInfo,
final ArrayList<int[]> bigramTargets, final ArrayList<int[]> bigramProbabilityInfo,
final ArrayList<int[]> shortcutTargets,
final ArrayList<Integer> shortcutProbabilities) {
mWord = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints);
mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo);
mShortcutTargets = CollectionUtils.newArrayList();
mBigrams = CollectionUtils.newArrayList();
mIsNotAWord = isNotAWord;
mIsBlacklistEntry = isBlacklisted;
mHasShortcuts = hasShortcuts;
mHasBigrams = hasBigram;
final int bigramTargetCount = bigramTargets.size();
for (int i = 0; i < bigramTargetCount; i++) {
final String bigramTargetString =
StringUtils.getStringFromNullTerminatedCodePointArray(bigramTargets.get(i));
mBigrams.add(new WeightedString(bigramTargetString,
createProbabilityInfoFromArray(bigramProbabilityInfo.get(i))));
}
final int shortcutTargetCount = shortcutTargets.size();
for (int i = 0; i < shortcutTargetCount; i++) {
final String shortcutTargetString =
StringUtils.getStringFromNullTerminatedCodePointArray(shortcutTargets.get(i));
mShortcutTargets.add(
new WeightedString(shortcutTargetString, shortcutProbabilities.get(i)));
}
}
public int getProbability() {
return mProbabilityInfo.mProbability;
}
private static int computeHashCode(WordProperty word) {
return Arrays.hashCode(new Object[] {
word.mWord,
word.mProbabilityInfo,
word.mShortcutTargets.hashCode(),
word.mBigrams.hashCode(),
word.mIsNotAWord,
word.mIsBlacklistEntry
});
}
/**
* Three-way comparison.
*
* A Word x is greater than a word y if x has a higher frequency. If they have the same
* frequency, they are sorted in lexicographic order.
*/
@Override
public int compareTo(final WordProperty w) {
if (getProbability() < w.getProbability()) return 1;
if (getProbability() > w.getProbability()) return -1;
return mWord.compareTo(w.mWord);
}
/**
* Equality test.
*
* Words are equal if they have the same frequency, the same spellings, and the same
* attributes.
*/
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (!(o instanceof WordProperty)) return false;
WordProperty w = (WordProperty)o;
return mProbabilityInfo.equals(w.mProbabilityInfo) && mWord.equals(w.mWord)
&& mShortcutTargets.equals(w.mShortcutTargets) && mBigrams.equals(w.mBigrams)
&& mIsNotAWord == w.mIsNotAWord && mIsBlacklistEntry == w.mIsBlacklistEntry
&& mHasBigrams == w.mHasBigrams && mHasShortcuts && w.mHasBigrams;
}
@Override
public int hashCode() {
if (mHashCode == 0) {
mHashCode = computeHashCode(this);
}
return mHashCode;
}
@UsedForTesting
public boolean isValid() {
return getProbability() != BinaryDictionary.NOT_A_PROBABILITY;
}
@Override
public String toString() {
// TODO: Move this logic to CombinedInputOutput.
final StringBuffer builder = new StringBuffer();
builder.append(" word=" + mWord);
builder.append(",");
builder.append(mProbabilityInfo.toString());
if (mIsNotAWord) {
builder.append(",");
builder.append("not_a_word=true");
}
if (mIsBlacklistEntry) {
builder.append(",");
builder.append("blacklisted=true");
}
builder.append("\n");
for (int i = 0; i < mBigrams.size(); i++) {
builder.append(" bigram=" + mBigrams.get(i).mWord);
builder.append(",");
builder.append(mBigrams.get(i).mProbabilityInfo.toString());
builder.append("\n");
}
for (int i = 0; i < mShortcutTargets.size(); i++) {
builder.append(" shortcut=" + mShortcutTargets.get(i).mWord);
builder.append(",");
builder.append(mShortcutTargets.get(i).mProbabilityInfo.toString());
builder.append("\n");
}
return builder.toString();
}
}

127
java/src/com/android/inputmethod/latin/utils/WordProperty.java
View File

@ -1,127 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.utils;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.ProbabilityInfo;
import java.util.ArrayList;
// This has information that belong to a unigram. This class has some detailed attributes such as
// historical information but they have to be checked only for testing purpose.
@UsedForTesting
public class WordProperty {
public final String mCodePoints;
public final boolean mIsNotAWord;
public final boolean mIsBlacklisted;
public final boolean mHasBigrams;
public final boolean mHasShortcuts;
public final ProbabilityInfo mProbabilityInfo;
public final ArrayList<WeightedString> mBigramTargets = CollectionUtils.newArrayList();
public final ArrayList<ProbabilityInfo> mBigramProbabilityInfo = CollectionUtils.newArrayList();
public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList();
private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) {
return new ProbabilityInfo(
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX],
probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX]);
}
// This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY.
public WordProperty(final int[] codePoints, final boolean isNotAWord,
final boolean isBlacklisted, final boolean hasBigram,
final boolean hasShortcuts, final int[] probabilityInfo,
final ArrayList<int[]> bigramTargets, final ArrayList<int[]> bigramProbabilityInfo,
final ArrayList<int[]> shortcutTargets,
final ArrayList<Integer> shortcutProbabilities) {
mCodePoints = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints);
mIsNotAWord = isNotAWord;
mIsBlacklisted = isBlacklisted;
mHasBigrams = hasBigram;
mHasShortcuts = hasShortcuts;
mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo);
final int bigramTargetCount = bigramTargets.size();
for (int i = 0; i < bigramTargetCount; i++) {
final String bigramTargetString =
StringUtils.getStringFromNullTerminatedCodePointArray(bigramTargets.get(i));
final ProbabilityInfo bigramProbability =
createProbabilityInfoFromArray(bigramProbabilityInfo.get(i));
mBigramTargets.add(
new WeightedString(bigramTargetString, bigramProbability.mProbability));
mBigramProbabilityInfo.add(bigramProbability);
}
final int shortcutTargetCount = shortcutTargets.size();
for (int i = 0; i < shortcutTargetCount; i++) {
final String shortcutTargetString =
StringUtils.getStringFromNullTerminatedCodePointArray(shortcutTargets.get(i));
mShortcutTargets.add(
new WeightedString(shortcutTargetString, shortcutProbabilities.get(i)));
}
}
@UsedForTesting
public boolean isValid() {
return mProbabilityInfo.mProbability != BinaryDictionary.NOT_A_PROBABILITY;
}
@Override
public String toString() {
// TODO: Move this logic to CombinedInputOutput.
final StringBuffer builder = new StringBuffer();
builder.append(" word=" + mCodePoints);
builder.append(",");
builder.append("f=" + mProbabilityInfo.mProbability);
if (mIsNotAWord) {
builder.append(",");
builder.append("not_a_word=true");
}
if (mIsBlacklisted) {
builder.append(",");
builder.append("blacklisted=true");
}
if (mProbabilityInfo.mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP) {
builder.append(",");
builder.append("historicalInfo=" + mProbabilityInfo);
}
builder.append("\n");
for (int i = 0; i < mBigramTargets.size(); i++) {
builder.append(" bigram=" + mBigramTargets.get(i).mWord);
builder.append(",");
builder.append("f=" + mBigramTargets.get(i).getProbability());
if (mBigramProbabilityInfo.get(i).mTimestamp
!= BinaryDictionary.NOT_A_VALID_TIMESTAMP) {
builder.append(",");
builder.append("historicalInfo=" + mBigramProbabilityInfo.get(i));
}
builder.append("\n");
}
for (int i = 0; i < mShortcutTargets.size(); i++) {
builder.append(" shortcut=" + mShortcutTargets.get(i).mWord);
builder.append(",");
builder.append("f=" + mShortcutTargets.get(i).getProbability());
builder.append("\n");
}
return builder.toString();
}
}

43
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -24,9 +24,9 @@ import android.util.Pair;
import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.WordProperty;
import java.io.File;
import java.io.IOException;
@ -916,15 +916,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
}
words.add(word);
wordProbabilities.put(word, unigramProbability);
final WordProperty unigramProperty = binaryDictionary.getWordProperty(word);
assertEquals(word, unigramProperty.mCodePoints);
assertTrue(unigramProperty.isValid());
assertEquals(isNotAWord, unigramProperty.mIsNotAWord);
assertEquals(isBlacklisted, unigramProperty.mIsBlacklisted);
assertEquals(false, unigramProperty.mHasBigrams);
assertEquals(false, unigramProperty.mHasShortcuts);
assertEquals(unigramProbability, unigramProperty.mProbabilityInfo.mProbability);
assertTrue(unigramProperty.mShortcutTargets.isEmpty());
final WordProperty wordProperty = binaryDictionary.getWordProperty(word);
assertEquals(word, wordProperty.mWord);
assertTrue(wordProperty.isValid());
assertEquals(isNotAWord, wordProperty.mIsNotAWord);
assertEquals(isBlacklisted, wordProperty.mIsBlacklistEntry);
assertEquals(false, wordProperty.mHasBigrams);
assertEquals(false, wordProperty.mHasShortcuts);
assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability);
assertTrue(wordProperty.mShortcutTargets.isEmpty());
}
for (int i = 0; i < BIGRAM_COUNT; i++) {
@ -955,18 +955,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
continue;
}
final HashSet<String> bigramWord1s = bigrams.get(word0);
final WordProperty unigramProperty = binaryDictionary.getWordProperty(word0);
assertEquals(bigramWord1s.size(), unigramProperty.mBigramTargets.size());
assertEquals(unigramProperty.mBigramTargets.size(),
unigramProperty.mBigramProbabilityInfo.size());
for (int j = 0; j < unigramProperty.mBigramTargets.size(); j++) {
final String word1 = unigramProperty.mBigramTargets.get(j).mWord;
final WordProperty wordProperty = binaryDictionary.getWordProperty(word0);
assertEquals(bigramWord1s.size(), wordProperty.mBigrams.size());
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int probability = unigramProperty.mBigramTargets.get(j).getProbability();
final int probability = wordProperty.mBigrams.get(j).getProbability();
assertEquals((int)bigramProbabilities.get(new Pair<String, String>(word0, word1)),
probability);
assertEquals(unigramProperty.mBigramProbabilityInfo.get(j).mProbability,
probability);
assertEquals(wordProperty.mBigrams.get(j).getProbability(), probability);
}
}
}
@ -1045,15 +1042,15 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final BinaryDictionary.GetNextWordPropertyResult result =
binaryDictionary.getNextWordProperty(token);
final WordProperty wordProperty = result.mWordProperty;
final String word0 = wordProperty.mCodePoints;
final String word0 = wordProperty.mWord;
assertEquals((int)wordProbabilitiesToCheckLater.get(word0),
wordProperty.mProbabilityInfo.mProbability);
wordSet.remove(word0);
final HashSet<String> bigramWord1s = bigrams.get(word0);
for (int j = 0; j < wordProperty.mBigramTargets.size(); j++) {
final String word1 = wordProperty.mBigramTargets.get(j).mWord;
for (int j = 0; j < wordProperty.mBigrams.size(); j++) {
final String word1 = wordProperty.mBigrams.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int probability = wordProperty.mBigramTargets.get(j).getProbability();
final int probability = wordProperty.mBigrams.get(j).getProbability();
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
assertEquals((int)bigramProbabilitiesToCheckLater.get(bigram), probability);
bigramSet.remove(bigram);

3
tools/dicttool/Android.mk
View File

@ -48,8 +48,7 @@ USED_TARGETTED_UTILS := \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/JniUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/LocaleUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/ResizableIntArray.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/StringUtils.java \
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/WordProperty.java
$(LATINIME_CORE_SOURCE_DIRECTORY)/utils/StringUtils.java
DICTTOOL_ONDEVICE_TESTS_DIRECTORY := \
$(LATINIME_LOCAL_DIR)/tests/src/com/android/inputmethod/latin/makedict/

37
tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
View File

@ -21,7 +21,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
import com.android.inputmethod.latin.makedict.WordProperty;
import java.io.BufferedReader;
import java.io.File;
@ -45,7 +45,7 @@ public class CombinedInputOutput {
private static final String DICTIONARY_TAG = "dictionary";
private static final String BIGRAM_TAG = "bigram";
private static final String SHORTCUT_TAG = "shortcut";
private static final String FREQUENCY_TAG = "f";
private static final String PROBABILITY_TAG = "f";
private static final String WORD_TAG = "word";
private static final String NOT_A_WORD_TAG = "not_a_word";
private static final String WHITELIST_TAG = "whitelist";
@ -138,7 +138,7 @@ public class CombinedInputOutput {
if (2 != params.length) throw new RuntimeException("Wrong format : " + line);
if (WORD_TAG.equals(params[0])) {
word = params[1];
} else if (FREQUENCY_TAG.equals(params[0])) {
} else if (PROBABILITY_TAG.equals(params[0])) {
freq = Integer.parseInt(params[1]);
} else if (NOT_A_WORD_TAG.equals(params[0])) {
isNotAWord = "true".equals(params[1]);
@ -152,7 +152,7 @@ public class CombinedInputOutput {
if (2 != params.length) throw new RuntimeException("Wrong format : " + line);
if (SHORTCUT_TAG.equals(params[0])) {
shortcut = params[1];
} else if (FREQUENCY_TAG.equals(params[0])) {
} else if (PROBABILITY_TAG.equals(params[0])) {
shortcutFreq = WHITELIST_TAG.equals(params[1])
? FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
: Integer.parseInt(params[1]);
@ -171,7 +171,7 @@ public class CombinedInputOutput {
if (2 != params.length) throw new RuntimeException("Wrong format : " + line);
if (BIGRAM_TAG.equals(params[0])) {
secondWordOfBigram = params[1];
} else if (FREQUENCY_TAG.equals(params[0])) {
} else if (PROBABILITY_TAG.equals(params[0])) {
bigramFreq = Integer.parseInt(params[1]);
}
}
@ -200,9 +200,10 @@ public class CombinedInputOutput {
*/
public static void writeDictionaryCombined(Writer destination, FusionDictionary dict)
throws IOException {
final TreeSet<Word> set = new TreeSet<Word>();
for (Word word : dict) {
set.add(word); // This for ordering by frequency, then by asciibetic order
final TreeSet<WordProperty> wordPropertiesInDict = new TreeSet<WordProperty>();
for (WordProperty wordProperty: dict) {
// This for ordering by frequency, then by asciibetic order
wordPropertiesInDict.add(wordProperty);
}
final HashMap<String, String> options = dict.mOptions.mAttributes;
destination.write(DICTIONARY_TAG + "=");
@ -215,20 +216,20 @@ public class CombinedInputOutput {
destination.write("," + key + "=" + value);
}
destination.write("\n");
for (Word word : set) {
destination.write(" " + WORD_TAG + "=" + word.mWord + ","
+ FREQUENCY_TAG + "=" + word.mFrequency
+ (word.mIsNotAWord ? "," + NOT_A_WORD_TAG + "=true\n" : "\n"));
if (null != word.mShortcutTargets) {
for (WeightedString target : word.mShortcutTargets) {
for (WordProperty wordProperty : wordPropertiesInDict) {
destination.write(" " + WORD_TAG + "=" + wordProperty.mWord + ","
+ PROBABILITY_TAG + "=" + wordProperty.getProbability()
+ (wordProperty.mIsNotAWord ? "," + NOT_A_WORD_TAG + "=true\n" : "\n"));
if (null != wordProperty.mShortcutTargets) {
for (WeightedString target : wordProperty.mShortcutTargets) {
destination.write(" " + SHORTCUT_TAG + "=" + target.mWord + ","
+ FREQUENCY_TAG + "=" + target.getProbability() + "\n");
+ PROBABILITY_TAG + "=" + target.getProbability() + "\n");
}
}
if (null != word.mBigrams) {
for (WeightedString bigram : word.mBigrams) {
if (null != wordProperty.mBigrams) {
for (WeightedString bigram : wordProperty.mBigrams) {
destination.write(" " + BIGRAM_TAG + "=" + bigram.mWord + ","
+ FREQUENCY_TAG + "=" + bigram.getProbability() + "\n");
+ PROBABILITY_TAG + "=" + bigram.getProbability() + "\n");
}
}
}

52
tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
View File

@ -19,7 +19,7 @@ package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
import com.android.inputmethod.latin.makedict.WordProperty;
import java.util.Arrays;
import java.util.ArrayList;
@ -108,42 +108,46 @@ public class Diff extends Dicttool.Command {
private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) {
boolean hasDifferences = false;
for (final Word word0 : dict0) {
final PtNode word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray,
word0.mWord);
if (null == word1) {
for (final WordProperty word0Property : dict0) {
final PtNode word1PtNode = FusionDictionary.findWordInTree(dict1.mRootNodeArray,
word0Property.mWord);
if (null == word1PtNode) {
// This word is not in dict1
System.out.println("Deleted: " + word0.mWord + " " + word0.mFrequency);
System.out.println("Deleted: " + word0Property.mWord + " "
+ word0Property.getProbability());
hasDifferences = true;
} else {
// We found the word. Compare frequencies, shortcuts, bigrams
if (word0.mFrequency != word1.getFrequency()) {
System.out.println("Freq changed: " + word0.mWord + " " + word0.mFrequency
+ " -> " + word1.getFrequency());
if (word0Property.getProbability() != word1PtNode.getFrequency()) {
System.out.println("Probability changed: " + word0Property.mWord + " "
+ word0Property.getProbability() + " -> " + word1PtNode.getFrequency());
hasDifferences = true;
}
if (word0.mIsNotAWord != word1.getIsNotAWord()) {
System.out.println("Not a word: " + word0.mWord + " " + word0.mIsNotAWord
+ " -> " + word1.getIsNotAWord());
if (word0Property.mIsNotAWord != word1PtNode.getIsNotAWord()) {
System.out.println("Not a word: " + word0Property.mWord + " "
+ word0Property.mIsNotAWord + " -> " + word1PtNode.getIsNotAWord());
hasDifferences = true;
}
if (word0.mIsBlacklistEntry != word1.getIsBlacklistEntry()) {
System.out.println("Blacklist: " + word0.mWord + " " + word0.mIsBlacklistEntry
+ " -> " + word1.getIsBlacklistEntry());
if (word0Property.mIsBlacklistEntry != word1PtNode.getIsBlacklistEntry()) {
System.out.println("Blacklist: " + word0Property.mWord + " "
+ word0Property.mIsBlacklistEntry + " -> "
+ word1PtNode.getIsBlacklistEntry());
hasDifferences = true;
}
hasDifferences |= hasAttributesDifferencesAndPrintThemIfAny(word0.mWord,
"Bigram", word0.mBigrams, word1.getBigrams());
hasDifferences |= hasAttributesDifferencesAndPrintThemIfAny(word0.mWord,
"Shortcut", word0.mShortcutTargets, word1.getShortcutTargets());
hasDifferences |= hasAttributesDifferencesAndPrintThemIfAny(word0Property.mWord,
"Bigram", word0Property.mBigrams, word1PtNode.getBigrams());
hasDifferences |= hasAttributesDifferencesAndPrintThemIfAny(word0Property.mWord,
"Shortcut", word0Property.mShortcutTargets,
word1PtNode.getShortcutTargets());
}
}
for (final Word word1 : dict1) {
final PtNode word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray,
word1.mWord);
if (null == word0) {
for (final WordProperty word1Property : dict1) {
final PtNode word0PtNode = FusionDictionary.findWordInTree(dict0.mRootNodeArray,
word1Property.mWord);
if (null == word0PtNode) {
// This word is not in dict0
System.out.println("Added: " + word1.mWord + " " + word1.mFrequency);
System.out.println("Added: " + word1Property.mWord + " "
+ word1Property.getProbability());
hasDifferences = true;
}
}

14
tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
View File

@ -20,7 +20,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
import com.android.inputmethod.latin.makedict.WordProperty;
import java.util.Arrays;
import java.util.ArrayList;
@ -43,14 +43,14 @@ public class Info extends Dicttool.Command {
int bigramCount = 0;
int shortcutCount = 0;
int whitelistCount = 0;
for (final Word w : dict) {
for (final WordProperty wordProperty : dict) {
++wordCount;
if (null != w.mBigrams) {
bigramCount += w.mBigrams.size();
if (null != wordProperty.mBigrams) {
bigramCount += wordProperty.mBigrams.size();
}
if (null != w.mShortcutTargets) {
shortcutCount += w.mShortcutTargets.size();
for (WeightedString shortcutTarget : w.mShortcutTargets) {
if (null != wordProperty.mShortcutTargets) {
shortcutCount += wordProperty.mShortcutTargets.size();
for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
if (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
== shortcutTarget.getProbability()) {
++whitelistCount;

33
tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
View File

@ -20,7 +20,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.Word;
import com.android.inputmethod.latin.makedict.WordProperty;
import java.io.BufferedReader;
import java.io.File;
@ -52,7 +52,7 @@ public class XmlDictInputOutput {
private static final String WORD_TAG = "w";
private static final String BIGRAM_TAG = "bigram";
private static final String SHORTCUT_TAG = "shortcut";
private static final String FREQUENCY_ATTR = "f";
private static final String PROBABILITY_ATTR = "f";
private static final String WORD_ATTR = "word";
private static final String NOT_A_WORD_ATTR = "not_a_word";
@ -107,7 +107,7 @@ public class XmlDictInputOutput {
mWord = "";
for (int attrIndex = 0; attrIndex < attrs.getLength(); ++attrIndex) {
final String attrName = attrs.getLocalName(attrIndex);
if (FREQUENCY_ATTR.equals(attrName)) {
if (PROBABILITY_ATTR.equals(attrName)) {
mFreq = Integer.parseInt(attrs.getValue(attrIndex));
}
}
@ -348,9 +348,9 @@ public class XmlDictInputOutput {
*/
public static void writeDictionaryXml(Writer destination, FusionDictionary dict)
throws IOException {
final TreeSet<Word> set = new TreeSet<Word>();
for (Word word : dict) {
set.add(word);
final TreeSet<WordProperty> wordPropertiesInDict = new TreeSet<WordProperty>();
for (WordProperty wordProperty : dict) {
wordPropertiesInDict.add(wordProperty);
}
// TODO: use an XMLSerializer if this gets big
destination.write("<wordlist format=\"2\"");
@ -361,23 +361,24 @@ public class XmlDictInputOutput {
}
destination.write(">\n");
destination.write("<!-- Warning: there is no code to read this format yet. -->\n");
for (Word word : set) {
destination.write(" <" + WORD_TAG + " " + WORD_ATTR + "=\"" + word.mWord + "\" "
+ FREQUENCY_ATTR + "=\"" + word.mFrequency
+ (word.mIsNotAWord ? "\" " + NOT_A_WORD_ATTR + "=\"true" : "") + "\">");
if (null != word.mShortcutTargets) {
for (WordProperty wordProperty : wordPropertiesInDict) {
destination.write(" <" + WORD_TAG + " " + WORD_ATTR + "=\"" + wordProperty.mWord
+ "\" " + PROBABILITY_ATTR + "=\"" + wordProperty.getProbability()
+ (wordProperty.mIsNotAWord ? "\" " + NOT_A_WORD_ATTR + "=\"true" : "")
+ "\">");
if (null != wordProperty.mShortcutTargets) {
destination.write("\n");
for (WeightedString target : word.mShortcutTargets) {
destination.write(" <" + SHORTCUT_TAG + " " + FREQUENCY_ATTR + "=\""
for (WeightedString target : wordProperty.mShortcutTargets) {
destination.write(" <" + SHORTCUT_TAG + " " + PROBABILITY_ATTR + "=\""
+ target.getProbability() + "\">" + target.mWord + "</" + SHORTCUT_TAG
+ ">\n");
}
destination.write(" ");
}
if (null != word.mBigrams) {
if (null != wordProperty.mBigrams) {
destination.write("\n");
for (WeightedString bigram : word.mBigrams) {
destination.write(" <" + BIGRAM_TAG + " " + FREQUENCY_ATTR + "=\""
for (WeightedString bigram : wordProperty.mBigrams) {
destination.write(" <" + BIGRAM_TAG + " " + PROBABILITY_ATTR + "=\""
+ bigram.getProbability() + "\">" + bigram.mWord
+ "</" + BIGRAM_TAG + ">\n");
}

6
tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
View File

@ -20,7 +20,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.Word;
import com.android.inputmethod.latin.makedict.WordProperty;
import junit.framework.TestCase;
@ -87,8 +87,8 @@ public class FusionDictionaryTest extends TestCase {
}
private void dumpDict(final FusionDictionary dict) {
for (Word w : dict) {
System.out.println("Word " + dumpWord(w.mWord));
for (WordProperty wordProperty : dict) {
System.out.println("Word " + dumpWord(wordProperty.mWord));
}
}