nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "[Refactor] Divide BinaryDictInputOutput into BinaryDictInputUtils and BinaryDictOutputUtils."

Browse source
This commit is contained in:
Ken Wakasa 2013-08-15 06:53:22 +00:00 committed by Android (Google) Code Review
commit f74ddfbb06
18 changed files with 940 additions and 892 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View file

@ -21,7 +21,7 @@ import android.content.SharedPreferences;
import android.content.res.AssetFileDescriptor; import android.content.res.AssetFileDescriptor;
import android.util.Log; import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils;
import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
@ -231,8 +231,8 @@ final public class BinaryDictionaryGetter {
try { try {
// Read the version of the file // Read the version of the file
inStream = new FileInputStream(f); inStream = new FileInputStream(f);
final BinaryDictInputOutput.ByteBufferWrapper buffer = final BinaryDictInputUtils.ByteBufferWrapper buffer =
new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( new BinaryDictInputUtils.ByteBufferWrapper(inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, f.length())); FileChannel.MapMode.READ_ONLY, 0, f.length()));
final int magic = buffer.readInt(); final int magic = buffer.readInt();
if (magic != FormatSpec.MAGIC_NUMBER) { if (magic != FormatSpec.MAGIC_NUMBER) {
@ -241,7 +241,7 @@ final public class BinaryDictionaryGetter {
final int formatVersion = buffer.readInt(); final int formatVersion = buffer.readInt();
final int headerSize = buffer.readInt(); final int headerSize = buffer.readInt();
final HashMap<String, String> options = CollectionUtils.newHashMap(); final HashMap<String, String> options = CollectionUtils.newHashMap();
BinaryDictInputOutput.populateOptions(buffer, headerSize, options); BinaryDictInputUtils.populateOptions(buffer, headerSize, options);
final String version = options.get(VERSION_KEY); final String version = options.get(VERSION_KEY);
if (null == version) { if (null == version) {

4
java/src/com/android/inputmethod/latin/DictionaryWriter.java
View file

@ -20,7 +20,7 @@ import android.content.Context;
import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictOutputUtils;
import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node; import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter {
@Override @Override
protected void writeBinaryDictionary(final FileOutputStream out) protected void writeBinaryDictionary(final FileOutputStream out)
throws IOException, UnsupportedFormatException { throws IOException, UnsupportedFormatException {
BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); BinaryDictOutputUtils.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
} }
@Override @Override

68
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View file

@ -18,8 +18,8 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
@ -58,6 +58,22 @@ public final class BinaryDictIOUtils {
} }
} }
static int getChildrenAddressSize(final int optionFlags,
final FormatOptions formatOptions) {
if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
return 1;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
return 2;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
return 3;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
default:
return 0;
}
}
/** /**
* Tours all node without recursive call. * Tours all node without recursive call.
*/ */
@ -86,7 +102,7 @@ public final class BinaryDictIOUtils {
if (index != p.mLength) index = p.mLength; if (index != p.mLength) index = p.mLength;
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer); p.mNumOfCharGroup = BinaryDictInputUtils.readCharGroupCount(buffer);
p.mAddress += getGroupCountSize(p.mNumOfCharGroup); p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
p.mPosition = 0; p.mPosition = 0;
} }
@ -94,8 +110,8 @@ public final class BinaryDictIOUtils {
stack.pop(); stack.pop();
continue; continue;
} }
CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer, CharGroupInfo info = BinaryDictInputUtils.readCharGroup(buffer, p.mAddress - headerSize,
p.mAddress - headerSize, formatOptions); formatOptions);
for (int i = 0; i < info.mCharacters.length; ++i) { for (int i = 0; i < info.mCharacters.length; ++i) {
pushedChars[index++] = info.mCharacters[i]; pushedChars[index++] = info.mCharacters[i];
} }
@ -153,7 +169,7 @@ public final class BinaryDictIOUtils {
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException { UnsupportedFormatException {
// Read header // Read header
final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer()); final FileHeader header = BinaryDictInputUtils.readHeader(reader.getBuffer());
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words, readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
frequencies, bigrams, header.mFormatOptions); frequencies, bigrams, header.mFormatOptions);
} }
@ -174,18 +190,18 @@ public final class BinaryDictIOUtils {
if (word == null) return FormatSpec.NOT_VALID_WORD; if (word == null) return FormatSpec.NOT_VALID_WORD;
if (buffer.position() != 0) buffer.position(0); if (buffer.position() != 0) buffer.position(0);
final FileHeader header = BinaryDictInputOutput.readHeader(buffer); final FileHeader header = BinaryDictInputUtils.readHeader(buffer);
int wordPos = 0; int wordPos = 0;
final int wordLen = word.codePointCount(0, word.length()); final int wordLen = word.codePointCount(0, word.length());
for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) {
if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD;
do { do {
final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); final int charGroupCount = BinaryDictInputUtils.readCharGroupCount(buffer);
boolean foundNextCharGroup = false; boolean foundNextCharGroup = false;
for (int i = 0; i < charGroupCount; ++i) { for (int i = 0; i < charGroupCount; ++i) {
final int charGroupPos = buffer.position(); final int charGroupPos = buffer.position();
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, final CharGroupInfo currentInfo = BinaryDictInputUtils.readCharGroup(buffer,
buffer.position(), header.mFormatOptions); buffer.position(), header.mFormatOptions);
final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags,
header.mFormatOptions); header.mFormatOptions);
@ -271,7 +287,7 @@ public final class BinaryDictIOUtils {
*/ */
private static int writeVariableAddress(final OutputStream destination, final int value) private static int writeVariableAddress(final OutputStream destination, final int value)
throws IOException { throws IOException {
switch (BinaryDictInputOutput.getByteSize(value)) { switch (BinaryDictOutputUtils.getByteSize(value)) {
case 1: case 1:
destination.write((byte)value); destination.write((byte)value);
break; break;
@ -285,15 +301,15 @@ public final class BinaryDictIOUtils {
destination.write((byte)(0xFF & value)); destination.write((byte)(0xFF & value));
break; break;
} }
return BinaryDictInputOutput.getByteSize(value); return BinaryDictOutputUtils.getByteSize(value);
} }
static void skipCharGroup(final FusionDictionaryBufferInterface buffer, static void skipCharGroup(final FusionDictionaryBufferInterface buffer,
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
final int flags = buffer.readUnsignedByte(); final int flags = buffer.readUnsignedByte();
BinaryDictInputOutput.readParentAddress(buffer, formatOptions); BinaryDictInputUtils.readParentAddress(buffer, formatOptions);
skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions); BinaryDictInputUtils.readChildrenAddress(buffer, flags, formatOptions);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
final int shortcutsSize = buffer.readUnsignedShort(); final int shortcutsSize = buffer.readUnsignedShort();
@ -411,14 +427,14 @@ public final class BinaryDictIOUtils {
if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) {
final int shortcutListSize = final int shortcutListSize =
BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); BinaryDictOutputUtils.getShortcutListSize(info.mShortcutTargets);
destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize >> 8));
destination.write((byte)(shortcutListSize & 0xFF)); destination.write((byte)(shortcutListSize & 0xFF));
size += 2; size += 2;
final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator();
while (shortcutIterator.hasNext()) { while (shortcutIterator.hasNext()) {
final WeightedString target = shortcutIterator.next(); final WeightedString target = shortcutIterator.next();
destination.write((byte)BinaryDictInputOutput.makeShortcutFlags( destination.write((byte)BinaryDictOutputUtils.makeShortcutFlags(
shortcutIterator.hasNext(), target.mFrequency)); shortcutIterator.hasNext(), target.mFrequency));
size++; size++;
size += writeString(destination, target.mWord); size += writeString(destination, target.mWord);
@ -427,7 +443,7 @@ public final class BinaryDictIOUtils {
if (info.mBigrams != null) { if (info.mBigrams != null) {
// TODO: Consolidate this code with the code that computes the size of the bigram list // TODO: Consolidate this code with the code that computes the size of the bigram list
// in BinaryDictionaryInputOutput#computeActualNodeSize // in BinaryDictionaryOutput#computeActualNodeSize
for (int i = 0; i < info.mBigrams.size(); ++i) { for (int i = 0; i < info.mBigrams.size(); ++i) {
final int bigramFrequency = info.mBigrams.get(i).mFrequency; final int bigramFrequency = info.mBigrams.get(i).mFrequency;
@ -437,7 +453,7 @@ public final class BinaryDictIOUtils {
final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress
+ size); + size);
bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0;
switch (BinaryDictInputOutput.getByteSize(bigramOffset)) { switch (BinaryDictOutputUtils.getByteSize(bigramOffset)) {
case 1: case 1:
bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
break; break;
@ -461,18 +477,18 @@ public final class BinaryDictIOUtils {
*/ */
static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) {
int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters) + BinaryDictOutputUtils.getGroupCharactersSize(info.mCharacters)
+ BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions); + getChildrenAddressSize(info.mFlags, formatOptions);
if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) {
size += FormatSpec.GROUP_FREQUENCY_SIZE; size += FormatSpec.GROUP_FREQUENCY_SIZE;
} }
if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) {
size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); size += BinaryDictOutputUtils.getShortcutListSize(info.mShortcutTargets);
} }
if (info.mBigrams != null) { if (info.mBigrams != null) {
for (final PendingAttribute attr : info.mBigrams) { for (final PendingAttribute attr : info.mBigrams) {
size += FormatSpec.GROUP_FLAGS_SIZE; size += FormatSpec.GROUP_FLAGS_SIZE;
size += BinaryDictInputOutput.getByteSize(attr.mAddress); size += BinaryDictOutputUtils.getByteSize(attr.mAddress);
} }
} }
return size; return size;
@ -520,9 +536,9 @@ public final class BinaryDictIOUtils {
int position = getTerminalPosition(buffer, word); int position = getTerminalPosition(buffer, word);
if (position != FormatSpec.NOT_VALID_WORD) { if (position != FormatSpec.NOT_VALID_WORD) {
buffer.position(0); buffer.position(0);
final FileHeader header = BinaryDictInputOutput.readHeader(buffer); final FileHeader header = BinaryDictInputUtils.readHeader(buffer);
buffer.position(position); buffer.position(position);
return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions); return BinaryDictInputUtils.readCharGroup(buffer, position, header.mFormatOptions);
} }
return null; return null;
} }
@ -544,10 +560,10 @@ public final class BinaryDictIOUtils {
final FileInputStream inStream = new FileInputStream(file); final FileInputStream inStream = new FileInputStream(file);
try { try {
inStream.read(buffer); inStream.read(buffer);
final BinaryDictInputOutput.ByteBufferWrapper wrapper = final BinaryDictInputUtils.ByteBufferWrapper wrapper =
new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( new BinaryDictInputUtils.ByteBufferWrapper(inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, offset, length)); FileChannel.MapMode.READ_ONLY, offset, length));
return BinaryDictInputOutput.readHeader(wrapper); return BinaryDictInputUtils.readHeader(wrapper);
} finally { } finally {
inStream.close(); inStream.close();
} }

811
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputUtils.java Normal file
View file

@ -0,0 +1,811 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
/**
* Reads binary files for a FusionDictionary.
*
* All the methods in this class are static.
*/
public final class BinaryDictInputUtils {
private BinaryDictInputUtils() {
// This utility class is not publicly instantiable.
}
private static final boolean DBG = MakedictLog.DBG;
private static final int MAX_JUMPS = 12;
@UsedForTesting
public interface FusionDictionaryBufferInterface {
public int readUnsignedByte();
public int readUnsignedShort();
public int readUnsignedInt24();
public int readInt();
public int position();
public void position(int newPosition);
public void put(final byte b);
public int limit();
@UsedForTesting
public int capacity();
}
public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
private ByteBuffer mBuffer;
public ByteBufferWrapper(final ByteBuffer buffer) {
mBuffer = buffer;
}
@Override
public int readUnsignedByte() {
return mBuffer.get() & 0xFF;
}
@Override
public int readUnsignedShort() {
return mBuffer.getShort() & 0xFFFF;
}
@Override
public int readUnsignedInt24() {
final int retval = readUnsignedByte();
return (retval << 16) + readUnsignedShort();
}
@Override
public int readInt() {
return mBuffer.getInt();
}
@Override
public int position() {
return mBuffer.position();
}
@Override
public void position(int newPos) {
mBuffer.position(newPos);
}
@Override
public void put(final byte b) {
mBuffer.put(b);
}
@Override
public int limit() {
return mBuffer.limit();
}
@Override
public int capacity() {
return mBuffer.capacity();
}
}
/**
* A class grouping utility function for our specific character encoding.
*/
static final class CharEncoding {
private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
/**
* Helper method to find out whether this code fits on one byte
*/
private static boolean fitsOnOneByte(final int character) {
return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
&& character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
}
/**
* Compute the size of a character given its character code.
*
* Char format is:
* 1 byte = bbbbbbbb match
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
* else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
* unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
* 00011111 would be outside unicode.
* else: iso-latin-1 code
* This allows for the whole unicode range to be encoded, including chars outside of
* the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
* characters which should never happen anyway (and still work, but take 3 bytes).
*
* @param character the character code.
* @return the size in binary encoded-form, either 1 or 3 bytes.
*/
static int getCharSize(final int character) {
// See char encoding in FusionDictionary.java
if (fitsOnOneByte(character)) return 1;
if (FormatSpec.INVALID_CHARACTER == character) return 1;
return 3;
}
/**
* Compute the byte size of a character array.
*/
static int getCharArraySize(final int[] chars) {
int size = 0;
for (int character : chars) size += getCharSize(character);
return size;
}
/**
* Writes a char array to a byte buffer.
*
* @param codePoints the code point array to write.
* @param buffer the byte buffer to write to.
* @param index the index in buffer to write the character array to.
* @return the index after the last character.
*/
static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
for (int codePoint : codePoints) {
if (1 == getCharSize(codePoint)) {
buffer[index++] = (byte)codePoint;
} else {
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
buffer[index++] = (byte)(0xFF & codePoint);
}
}
return index;
}
/**
* Writes a string with our character format to a byte buffer.
*
* This will also write the terminator byte.
*
* @param buffer the byte buffer to write to.
* @param origin the offset to write from.
* @param word the string to write.
* @return the size written, in bytes.
*/
static int writeString(final byte[] buffer, final int origin,
final String word) {
final int length = word.length();
int index = origin;
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
final int codePoint = word.codePointAt(i);
if (1 == getCharSize(codePoint)) {
buffer[index++] = (byte)codePoint;
} else {
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
buffer[index++] = (byte)(0xFF & codePoint);
}
}
buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
return index - origin;
}
/**
* Writes a string with our character format to a ByteArrayOutputStream.
*
* This will also write the terminator byte.
*
* @param buffer the ByteArrayOutputStream to write to.
* @param word the string to write.
*/
static void writeString(final ByteArrayOutputStream buffer, final String word) {
final int length = word.length();
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
final int codePoint = word.codePointAt(i);
if (1 == getCharSize(codePoint)) {
buffer.write((byte) codePoint);
} else {
buffer.write((byte) (0xFF & (codePoint >> 16)));
buffer.write((byte) (0xFF & (codePoint >> 8)));
buffer.write((byte) (0xFF & codePoint));
}
}
buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
}
/**
* Reads a string from a buffer. This is the converse of the above method.
*/
private static String readString(final FusionDictionaryBufferInterface buffer) {
final StringBuilder s = new StringBuilder();
int character = readChar(buffer);
while (character != FormatSpec.INVALID_CHARACTER) {
s.appendCodePoint(character);
character = readChar(buffer);
}
return s.toString();
}
/**
* Reads a character from the buffer.
*
* This follows the character format documented earlier in this source file.
*
* @param buffer the buffer, positioned over an encoded character.
* @return the character code.
*/
static int readChar(final FusionDictionaryBufferInterface buffer) {
int character = buffer.readUnsignedByte();
if (!fitsOnOneByte(character)) {
if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
return FormatSpec.INVALID_CHARACTER;
}
character <<= 16;
character += buffer.readUnsignedShort();
}
return character;
}
}
// Input methods: Read a binary dictionary to memory.
// readDictionaryBinary is the public entry point for them.
static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
final int optionFlags, final FormatOptions options) {
if (options.mSupportsDynamicUpdate) {
final int address = buffer.readUnsignedInt24();
if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
if ((address & FormatSpec.MSB24) != 0) {
return -(address & FormatSpec.SINT24_MAX);
} else {
return address;
}
}
int address;
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
return buffer.readUnsignedByte();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
return buffer.readUnsignedShort();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
return buffer.readUnsignedInt24();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
default:
return FormatSpec.NO_CHILDREN_ADDRESS;
}
}
static int readParentAddress(final FusionDictionaryBufferInterface buffer,
final FormatOptions formatOptions) {
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
final int parentAddress = buffer.readUnsignedInt24();
final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
return sign * (parentAddress & FormatSpec.SINT24_MAX);
} else {
return FormatSpec.NO_PARENT_ADDRESS;
}
}
private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
final int originalGroupAddress, final FormatOptions options) {
int addressPointer = originalGroupAddress;
final int flags = buffer.readUnsignedByte();
++addressPointer;
final int parentAddress = readParentAddress(buffer, options);
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
addressPointer += 3;
}
final int characters[];
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
int index = 0;
int character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
while (-1 != character) {
// FusionDictionary is making sure that the length of the word is smaller than
// MAX_WORD_LENGTH.
// So we'll never write past the end of CHARACTER_BUFFER.
CHARACTER_BUFFER[index++] = character;
character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
}
characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
} else {
final int character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
characters = new int[] { character };
}
final int frequency;
if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
++addressPointer;
frequency = buffer.readUnsignedByte();
} else {
frequency = CharGroup.NOT_A_TERMINAL;
}
int childrenAddress = readChildrenAddress(buffer, flags, options);
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
childrenAddress += addressPointer;
}
addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
ArrayList<WeightedString> shortcutTargets = null;
if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
final int pointerBefore = buffer.position();
shortcutTargets = new ArrayList<WeightedString>();
buffer.readUnsignedShort(); // Skip the size
while (true) {
final int targetFlags = buffer.readUnsignedByte();
final String word = CharEncoding.readString(buffer);
shortcutTargets.add(new WeightedString(word,
targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
}
addressPointer += buffer.position() - pointerBefore;
}
ArrayList<PendingAttribute> bigrams = null;
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>();
int bigramCount = 0;
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
final int bigramFlags = buffer.readUnsignedByte();
++addressPointer;
final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
? 1 : -1;
int bigramAddress = addressPointer;
switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
bigramAddress += sign * buffer.readUnsignedByte();
addressPointer += 1;
break;
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
bigramAddress += sign * buffer.readUnsignedShort();
addressPointer += 2;
break;
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
final int offset = (buffer.readUnsignedByte() << 16)
+ buffer.readUnsignedShort();
bigramAddress += sign * offset;
addressPointer += 3;
break;
default:
throw new RuntimeException("Has bigrams with no address");
}
bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
bigramAddress));
if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
}
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
MakedictLog.d("too many bigrams in a group.");
}
}
return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
parentAddress, childrenAddress, shortcutTargets, bigrams);
}
/**
* Reads and returns the char group count out of a buffer and forwards the pointer.
*/
public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
final int msb = buffer.readUnsignedByte();
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
return msb;
} else {
return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
+ buffer.readUnsignedByte();
}
}
// The word cache here is a stopgap bandaid to help the catastrophic performance
// of this method. Since it performs direct, unbuffered random access to the file and
// may be called hundreds of thousands of times, the resulting performance is not
// reasonable without some kind of cache. Thus:
private static TreeMap<Integer, WeightedString> wordCache =
new TreeMap<Integer, WeightedString>();
/**
* Finds, as a string, the word at the address passed as an argument.
*
* @param buffer the buffer to read from.
* @param headerSize the size of the header.
* @param address the address to seek.
* @param formatOptions file format options.
* @return the word with its frequency, as a weighted string.
*/
/* package for tests */ static WeightedString getWordAtAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions formatOptions) {
final WeightedString cachedString = wordCache.get(address);
if (null != cachedString) return cachedString;
final WeightedString result;
final int originalPointer = buffer.position();
buffer.position(address);
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
} else {
result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
formatOptions);
}
wordCache.put(address, result);
buffer.position(originalPointer);
return result;
}
// TODO: static!? This will behave erratically when used in multi-threaded code.
// We need to fix this
private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
@SuppressWarnings("unused")
private static WeightedString getWordAtAddressWithParentAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions options) {
int currentAddress = address;
int index = FormatSpec.MAX_WORD_LENGTH - 1;
int frequency = Integer.MIN_VALUE;
// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
CharGroupInfo currentInfo;
int loopCounter = 0;
do {
buffer.position(currentAddress + headerSize);
currentInfo = readCharGroup(buffer, currentAddress, options);
if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
if (DBG && loopCounter++ > MAX_JUMPS) {
MakedictLog.d("Too many jumps - probably a bug");
}
} while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
sGetWordBuffer[index--] =
currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
}
if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
return new WeightedString(
new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
frequency);
}
private static WeightedString getWordAtAddressWithoutParentAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions options) {
buffer.position(headerSize);
final int count = readCharGroupCount(buffer);
int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
final StringBuilder builder = new StringBuilder();
WeightedString result = null;
CharGroupInfo last = null;
for (int i = count - 1; i >= 0; --i) {
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
groupOffset = info.mEndAddress;
if (info.mOriginalAddress == address) {
builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
result = new WeightedString(builder.toString(), info.mFrequency);
break; // and return
}
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
if (info.mChildrenAddress > address) {
if (null == last) continue;
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize);
i = readCharGroupCount(buffer);
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
last = null;
continue;
}
last = info;
}
if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize);
i = readCharGroupCount(buffer);
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
last = null;
continue;
}
}
return result;
}
/**
* Reads a single node from a buffer.
*
* This methods reads the file at the current position. A node is fully expected to start at
* the current position.
* This will recursively read other nodes into the structure, populating the reverse
* maps on the fly and using them to keep track of already read nodes.
*
* @param buffer the buffer, correctly positioned at the start of a node.
* @param headerSize the size, in bytes, of the file header.
* @param reverseNodeMap a mapping from addresses to already read nodes.
* @param reverseGroupMap a mapping from addresses to already read character groups.
* @param options file format options.
* @return the read node with all his children already read.
*/
private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
final FormatOptions options)
throws IOException {
final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
final int nodeOrigin = buffer.position() - headerSize;
do { // Scan the linked-list node.
final int nodeHeadPosition = buffer.position() - headerSize;
final int count = readCharGroupCount(buffer);
int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
ArrayList<WeightedString> bigrams = null;
if (null != info.mBigrams) {
bigrams = new ArrayList<WeightedString>();
for (PendingAttribute bigram : info.mBigrams) {
final WeightedString word = getWordAtAddress(
buffer, headerSize, bigram.mAddress, options);
final int reconstructedFrequency =
reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
}
}
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
Node children = reverseNodeMap.get(info.mChildrenAddress);
if (null == children) {
final int currentPosition = buffer.position();
buffer.position(info.mChildrenAddress + headerSize);
children = readNode(
buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
buffer.position(currentPosition);
}
nodeContents.add(
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
info.mFrequency,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
} else {
nodeContents.add(
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
info.mFrequency,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
}
groupOffset = info.mEndAddress;
}
// reach the end of the array.
if (options.mSupportsDynamicUpdate) {
final int nextAddress = buffer.readUnsignedInt24();
if (nextAddress >= 0 && nextAddress < buffer.limit()) {
buffer.position(nextAddress);
} else {
break;
}
}
} while (options.mSupportsDynamicUpdate &&
buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
final Node node = new Node(nodeContents);
node.mCachedAddressBeforeUpdate = nodeOrigin;
node.mCachedAddressAfterUpdate = nodeOrigin;
reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
return node;
}
/**
* Helper function to get the binary format version from the header.
* @throws IOException
*/
private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
throws IOException {
final int magic = buffer.readInt();
if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
return FormatSpec.NOT_A_VERSION_NUMBER;
}
/**
* Helper function to get and validate the binary format version.
* @throws UnsupportedFormatException
* @throws IOException
*/
private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
final int version = getFormatVersion(buffer);
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
throw new UnsupportedFormatException("This file has version " + version
+ ", but this implementation does not support versions above "
+ FormatSpec.MAXIMUM_SUPPORTED_VERSION);
}
return version;
}
/**
* Reads a header from a buffer.
* @param buffer the buffer to read.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
final int version = checkFormatVersion(buffer);
final int optionsFlags = buffer.readUnsignedShort();
final HashMap<String, String> attributes = new HashMap<String, String>();
final int headerSize;
headerSize = buffer.readInt();
if (headerSize < 0) {
throw new UnsupportedFormatException("header size can't be negative.");
}
populateOptions(buffer, headerSize, attributes);
buffer.position(headerSize);
final FileHeader header = new FileHeader(headerSize,
new FusionDictionary.DictionaryOptions(attributes,
0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
new FormatOptions(version,
0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
return header;
}
/**
* Reads options from a buffer and populate a map with their contents.
*
* The buffer is read at the current position, so the caller must take care the pointer
* is in the right place before calling this.
*/
public static void populateOptions(final FusionDictionaryBufferInterface buffer,
final int headerSize, final HashMap<String, String> options) {
while (buffer.position() < headerSize) {
final String key = CharEncoding.readString(buffer);
final String value = CharEncoding.readString(buffer);
options.put(key, value);
}
}
/**
* Reads a buffer and returns the memory representation of the dictionary.
*
* This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param reader the reader.
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// clear cache
wordCache.clear();
// if the buffer has not been opened, open the buffer with bytebuffer.
if (reader.getBuffer() == null) reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
if (reader.getBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader header = readHeader(reader.getBuffer());
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
reverseGroupMapping, header.mFormatOptions);
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
if (null != dict) {
for (final Word w : dict) {
if (w.mIsBlacklistEntry) {
newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
} else {
newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
}
}
for (final Word w : dict) {
// By construction a binary dictionary may not have bigrams pointing to
// words that are not also registered as unigrams so we don't have to avoid
// them explicitly here.
for (final WeightedString bigram : w.mBigrams) {
newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
}
}
}
return newDict;
}
/**
* Helper method to pass a file name instead of a File object to isBinaryDictionary.
*/
public static boolean isBinaryDictionary(final String filename) {
final File file = new File(filename);
return isBinaryDictionary(file);
}
/**
* Basic test to find out whether the file is a binary dictionary or not.
*
* Concretely this only tests the magic number.
*
* @param file The file to test.
* @return true if it's a binary dictionary, false otherwise
*/
public static boolean isBinaryDictionary(final File file) {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
final int version = getFormatVersion(new ByteBufferWrapper(buffer));
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
} catch (FileNotFoundException e) {
return false;
} catch (IOException e) {
return false;
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
}
/**
* Calculate bigram frequency from compressed value
*
* @see #BinaryDictOutput.makeBigramFlags
*
* @param unigramFrequency
* @param bigramFrequency compressed frequency
* @return approximate bigram frequency
*/
public static int reconstructBigramFrequency(final int unigramFrequency,
final int bigramFrequency) {
final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
/ (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
return (int)resultFreqFloat;
}
}

828
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java → java/src/com/android/inputmethod/latin/makedict/BinaryDictOutputUtils.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2011 The Android Open Source Project * Copyright (C) 2013 The Android Open Source Project
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -16,8 +16,7 @@
package com.android.inputmethod.latin.makedict; package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
@ -25,26 +24,21 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
/** /**
* Reads and writes XML files for a FusionDictionary. * Writes binary files for a FusionDictionary.
* *
* All the methods in this class are static. * All the methods in this class are static.
*/ */
public final class BinaryDictInputOutput { public class BinaryDictOutputUtils {
private BinaryDictOutputUtils() {
// This utility class is not publicly instantiable.
}
private static final boolean DBG = MakedictLog.DBG; private static final boolean DBG = MakedictLog.DBG;
@ -54,229 +48,6 @@ public final class BinaryDictInputOutput {
// If the number of passes exceeds this number, makedict bails with an exception on // If the number of passes exceeds this number, makedict bails with an exception on
// suspicion that a bug might be causing an infinite loop. // suspicion that a bug might be causing an infinite loop.
private static final int MAX_PASSES = 24; private static final int MAX_PASSES = 24;
private static final int MAX_JUMPS = 12;
@UsedForTesting
public interface FusionDictionaryBufferInterface {
public int readUnsignedByte();
public int readUnsignedShort();
public int readUnsignedInt24();
public int readInt();
public int position();
public void position(int newPosition);
public void put(final byte b);
public int limit();
@UsedForTesting
public int capacity();
}
public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
private ByteBuffer mBuffer;
public ByteBufferWrapper(final ByteBuffer buffer) {
mBuffer = buffer;
}
@Override
public int readUnsignedByte() {
return mBuffer.get() & 0xFF;
}
@Override
public int readUnsignedShort() {
return mBuffer.getShort() & 0xFFFF;
}
@Override
public int readUnsignedInt24() {
final int retval = readUnsignedByte();
return (retval << 16) + readUnsignedShort();
}
@Override
public int readInt() {
return mBuffer.getInt();
}
@Override
public int position() {
return mBuffer.position();
}
@Override
public void position(int newPos) {
mBuffer.position(newPos);
}
@Override
public void put(final byte b) {
mBuffer.put(b);
}
@Override
public int limit() {
return mBuffer.limit();
}
@Override
public int capacity() {
return mBuffer.capacity();
}
}
/**
* A class grouping utility function for our specific character encoding.
*/
static final class CharEncoding {
private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
/**
* Helper method to find out whether this code fits on one byte
*/
private static boolean fitsOnOneByte(final int character) {
return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
&& character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
}
/**
* Compute the size of a character given its character code.
*
* Char format is:
* 1 byte = bbbbbbbb match
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
* else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
* unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
* 00011111 would be outside unicode.
* else: iso-latin-1 code
* This allows for the whole unicode range to be encoded, including chars outside of
* the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
* characters which should never happen anyway (and still work, but take 3 bytes).
*
* @param character the character code.
* @return the size in binary encoded-form, either 1 or 3 bytes.
*/
static int getCharSize(final int character) {
// See char encoding in FusionDictionary.java
if (fitsOnOneByte(character)) return 1;
if (FormatSpec.INVALID_CHARACTER == character) return 1;
return 3;
}
/**
* Compute the byte size of a character array.
*/
private static int getCharArraySize(final int[] chars) {
int size = 0;
for (int character : chars) size += getCharSize(character);
return size;
}
/**
* Writes a char array to a byte buffer.
*
* @param codePoints the code point array to write.
* @param buffer the byte buffer to write to.
* @param index the index in buffer to write the character array to.
* @return the index after the last character.
*/
private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
for (int codePoint : codePoints) {
if (1 == getCharSize(codePoint)) {
buffer[index++] = (byte)codePoint;
} else {
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
buffer[index++] = (byte)(0xFF & codePoint);
}
}
return index;
}
/**
* Writes a string with our character format to a byte buffer.
*
* This will also write the terminator byte.
*
* @param buffer the byte buffer to write to.
* @param origin the offset to write from.
* @param word the string to write.
* @return the size written, in bytes.
*/
private static int writeString(final byte[] buffer, final int origin,
final String word) {
final int length = word.length();
int index = origin;
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
final int codePoint = word.codePointAt(i);
if (1 == getCharSize(codePoint)) {
buffer[index++] = (byte)codePoint;
} else {
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
buffer[index++] = (byte)(0xFF & codePoint);
}
}
buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
return index - origin;
}
/**
* Writes a string with our character format to a ByteArrayOutputStream.
*
* This will also write the terminator byte.
*
* @param buffer the ByteArrayOutputStream to write to.
* @param word the string to write.
*/
private static void writeString(final ByteArrayOutputStream buffer, final String word) {
final int length = word.length();
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
final int codePoint = word.codePointAt(i);
if (1 == getCharSize(codePoint)) {
buffer.write((byte) codePoint);
} else {
buffer.write((byte) (0xFF & (codePoint >> 16)));
buffer.write((byte) (0xFF & (codePoint >> 8)));
buffer.write((byte) (0xFF & codePoint));
}
}
buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
}
/**
* Reads a string from a buffer. This is the converse of the above method.
*/
private static String readString(final FusionDictionaryBufferInterface buffer) {
final StringBuilder s = new StringBuilder();
int character = readChar(buffer);
while (character != FormatSpec.INVALID_CHARACTER) {
s.appendCodePoint(character);
character = readChar(buffer);
}
return s.toString();
}
/**
* Reads a character from the buffer.
*
* This follows the character format documented earlier in this source file.
*
* @param buffer the buffer, positioned over an encoded character.
* @return the character code.
*/
static int readChar(final FusionDictionaryBufferInterface buffer) {
int character = buffer.readUnsignedByte();
if (!fitsOnOneByte(character)) {
if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
return FormatSpec.INVALID_CHARACTER;
}
character <<= 16;
character += buffer.readUnsignedShort();
}
return character;
}
}
/** /**
* Compute the binary size of the character array. * Compute the binary size of the character array.
@ -345,6 +116,21 @@ public final class BinaryDictInputOutput {
return size; return size;
} }
/**
* Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
*
* @param group the group of which to compute the size of the header
* @param options file format options.
*/
private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ getGroupCharactersSize(group);
} else {
return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
}
}
/** /**
* Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything.
* *
@ -386,21 +172,6 @@ public final class BinaryDictInputOutput {
node.mCachedSize = size; node.mCachedSize = size;
} }
/**
* Compute the size of the header (flag + [parent address] + characters size) of a CharGroup.
*
* @param group the group of which to compute the size of the header
* @param options file format options.
*/
private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) {
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ getGroupCharactersSize(group);
} else {
return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group);
}
}
/** /**
* Compute the size, in bytes, that an address will occupy. * Compute the size, in bytes, that an address will occupy.
* *
@ -1219,557 +990,4 @@ public final class BinaryDictInputOutput {
destination.close(); destination.close();
MakedictLog.i("Done"); MakedictLog.i("Done");
} }
// Input methods: Read a binary dictionary to memory.
// readDictionaryBinary is the public entry point for them.
static int getChildrenAddressSize(final int optionFlags,
final FormatOptions formatOptions) {
if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
return 1;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
return 2;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
return 3;
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
default:
return 0;
}
}
static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
final int optionFlags, final FormatOptions options) {
if (options.mSupportsDynamicUpdate) {
final int address = buffer.readUnsignedInt24();
if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
if ((address & FormatSpec.MSB24) != 0) {
return -(address & FormatSpec.SINT24_MAX);
} else {
return address;
}
}
int address;
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
return buffer.readUnsignedByte();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
return buffer.readUnsignedShort();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
return buffer.readUnsignedInt24();
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
default:
return FormatSpec.NO_CHILDREN_ADDRESS;
}
}
static int readParentAddress(final FusionDictionaryBufferInterface buffer,
final FormatOptions formatOptions) {
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
final int parentAddress = buffer.readUnsignedInt24();
final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
return sign * (parentAddress & FormatSpec.SINT24_MAX);
} else {
return FormatSpec.NO_PARENT_ADDRESS;
}
}
private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
final int originalGroupAddress, final FormatOptions options) {
int addressPointer = originalGroupAddress;
final int flags = buffer.readUnsignedByte();
++addressPointer;
final int parentAddress = readParentAddress(buffer, options);
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
addressPointer += 3;
}
final int characters[];
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
int index = 0;
int character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
while (-1 != character) {
// FusionDictionary is making sure that the length of the word is smaller than
// MAX_WORD_LENGTH.
// So we'll never write past the end of CHARACTER_BUFFER.
CHARACTER_BUFFER[index++] = character;
character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
}
characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
} else {
final int character = CharEncoding.readChar(buffer);
addressPointer += CharEncoding.getCharSize(character);
characters = new int[] { character };
}
final int frequency;
if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
++addressPointer;
frequency = buffer.readUnsignedByte();
} else {
frequency = CharGroup.NOT_A_TERMINAL;
}
int childrenAddress = readChildrenAddress(buffer, flags, options);
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
childrenAddress += addressPointer;
}
addressPointer += getChildrenAddressSize(flags, options);
ArrayList<WeightedString> shortcutTargets = null;
if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
final int pointerBefore = buffer.position();
shortcutTargets = new ArrayList<WeightedString>();
buffer.readUnsignedShort(); // Skip the size
while (true) {
final int targetFlags = buffer.readUnsignedByte();
final String word = CharEncoding.readString(buffer);
shortcutTargets.add(new WeightedString(word,
targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
}
addressPointer += buffer.position() - pointerBefore;
}
ArrayList<PendingAttribute> bigrams = null;
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>();
int bigramCount = 0;
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
final int bigramFlags = buffer.readUnsignedByte();
++addressPointer;
final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
? 1 : -1;
int bigramAddress = addressPointer;
switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
bigramAddress += sign * buffer.readUnsignedByte();
addressPointer += 1;
break;
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
bigramAddress += sign * buffer.readUnsignedShort();
addressPointer += 2;
break;
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
final int offset = (buffer.readUnsignedByte() << 16)
+ buffer.readUnsignedShort();
bigramAddress += sign * offset;
addressPointer += 3;
break;
default:
throw new RuntimeException("Has bigrams with no address");
}
bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
bigramAddress));
if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
}
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
MakedictLog.d("too many bigrams in a group.");
}
}
return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
parentAddress, childrenAddress, shortcutTargets, bigrams);
}
/**
* Reads and returns the char group count out of a buffer and forwards the pointer.
*/
public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
final int msb = buffer.readUnsignedByte();
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
return msb;
} else {
return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
+ buffer.readUnsignedByte();
}
}
// The word cache here is a stopgap bandaid to help the catastrophic performance
// of this method. Since it performs direct, unbuffered random access to the file and
// may be called hundreds of thousands of times, the resulting performance is not
// reasonable without some kind of cache. Thus:
private static TreeMap<Integer, WeightedString> wordCache =
new TreeMap<Integer, WeightedString>();
/**
* Finds, as a string, the word at the address passed as an argument.
*
* @param buffer the buffer to read from.
* @param headerSize the size of the header.
* @param address the address to seek.
* @param formatOptions file format options.
* @return the word with its frequency, as a weighted string.
*/
/* package for tests */ static WeightedString getWordAtAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions formatOptions) {
final WeightedString cachedString = wordCache.get(address);
if (null != cachedString) return cachedString;
final WeightedString result;
final int originalPointer = buffer.position();
buffer.position(address);
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
} else {
result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
formatOptions);
}
wordCache.put(address, result);
buffer.position(originalPointer);
return result;
}
// TODO: static!? This will behave erratically when used in multi-threaded code.
// We need to fix this
private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
@SuppressWarnings("unused")
private static WeightedString getWordAtAddressWithParentAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions options) {
int currentAddress = address;
int index = FormatSpec.MAX_WORD_LENGTH - 1;
int frequency = Integer.MIN_VALUE;
// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
CharGroupInfo currentInfo;
int loopCounter = 0;
do {
buffer.position(currentAddress + headerSize);
currentInfo = readCharGroup(buffer, currentAddress, options);
if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
if (DBG && loopCounter++ > MAX_JUMPS) {
MakedictLog.d("Too many jumps - probably a bug");
}
} while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
sGetWordBuffer[index--] =
currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
}
if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
return new WeightedString(
new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
frequency);
}
private static WeightedString getWordAtAddressWithoutParentAddress(
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
final FormatOptions options) {
buffer.position(headerSize);
final int count = readCharGroupCount(buffer);
int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
final StringBuilder builder = new StringBuilder();
WeightedString result = null;
CharGroupInfo last = null;
for (int i = count - 1; i >= 0; --i) {
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
groupOffset = info.mEndAddress;
if (info.mOriginalAddress == address) {
builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
result = new WeightedString(builder.toString(), info.mFrequency);
break; // and return
}
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
if (info.mChildrenAddress > address) {
if (null == last) continue;
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize);
i = readCharGroupCount(buffer);
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
last = null;
continue;
}
last = info;
}
if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize);
i = readCharGroupCount(buffer);
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
last = null;
continue;
}
}
return result;
}
/**
* Reads a single node from a buffer.
*
* This methods reads the file at the current position. A node is fully expected to start at
* the current position.
* This will recursively read other nodes into the structure, populating the reverse
* maps on the fly and using them to keep track of already read nodes.
*
* @param buffer the buffer, correctly positioned at the start of a node.
* @param headerSize the size, in bytes, of the file header.
* @param reverseNodeMap a mapping from addresses to already read nodes.
* @param reverseGroupMap a mapping from addresses to already read character groups.
* @param options file format options.
* @return the read node with all his children already read.
*/
private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
final FormatOptions options)
throws IOException {
final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
final int nodeOrigin = buffer.position() - headerSize;
do { // Scan the linked-list node.
final int nodeHeadPosition = buffer.position() - headerSize;
final int count = readCharGroupCount(buffer);
int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
ArrayList<WeightedString> bigrams = null;
if (null != info.mBigrams) {
bigrams = new ArrayList<WeightedString>();
for (PendingAttribute bigram : info.mBigrams) {
final WeightedString word = getWordAtAddress(
buffer, headerSize, bigram.mAddress, options);
final int reconstructedFrequency =
reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
}
}
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
Node children = reverseNodeMap.get(info.mChildrenAddress);
if (null == children) {
final int currentPosition = buffer.position();
buffer.position(info.mChildrenAddress + headerSize);
children = readNode(
buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
buffer.position(currentPosition);
}
nodeContents.add(
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
info.mFrequency,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
} else {
nodeContents.add(
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
info.mFrequency,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
}
groupOffset = info.mEndAddress;
}
// reach the end of the array.
if (options.mSupportsDynamicUpdate) {
final int nextAddress = buffer.readUnsignedInt24();
if (nextAddress >= 0 && nextAddress < buffer.limit()) {
buffer.position(nextAddress);
} else {
break;
}
}
} while (options.mSupportsDynamicUpdate &&
buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
final Node node = new Node(nodeContents);
node.mCachedAddressBeforeUpdate = nodeOrigin;
node.mCachedAddressAfterUpdate = nodeOrigin;
reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
return node;
}
/**
* Helper function to get the binary format version from the header.
* @throws IOException
*/
private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
throws IOException {
final int magic = buffer.readInt();
if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
return FormatSpec.NOT_A_VERSION_NUMBER;
}
/**
* Helper function to get and validate the binary format version.
* @throws UnsupportedFormatException
* @throws IOException
*/
private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
final int version = getFormatVersion(buffer);
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
throw new UnsupportedFormatException("This file has version " + version
+ ", but this implementation does not support versions above "
+ FormatSpec.MAXIMUM_SUPPORTED_VERSION);
}
return version;
}
/**
* Reads a header from a buffer.
* @param buffer the buffer to read.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
final int version = checkFormatVersion(buffer);
final int optionsFlags = buffer.readUnsignedShort();
final HashMap<String, String> attributes = new HashMap<String, String>();
final int headerSize;
headerSize = buffer.readInt();
if (headerSize < 0) {
throw new UnsupportedFormatException("header size can't be negative.");
}
populateOptions(buffer, headerSize, attributes);
buffer.position(headerSize);
final FileHeader header = new FileHeader(headerSize,
new FusionDictionary.DictionaryOptions(attributes,
0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
new FormatOptions(version,
0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
return header;
}
/**
* Reads options from a buffer and populate a map with their contents.
*
* The buffer is read at the current position, so the caller must take care the pointer
* is in the right place before calling this.
*/
public static void populateOptions(final FusionDictionaryBufferInterface buffer,
final int headerSize, final HashMap<String, String> options) {
while (buffer.position() < headerSize) {
final String key = CharEncoding.readString(buffer);
final String value = CharEncoding.readString(buffer);
options.put(key, value);
}
}
/**
* Reads a buffer and returns the memory representation of the dictionary.
*
* This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param reader the reader.
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// clear cache
wordCache.clear();
// if the buffer has not been opened, open the buffer with bytebuffer.
if (reader.getBuffer() == null) reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
if (reader.getBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader header = readHeader(reader.getBuffer());
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
reverseGroupMapping, header.mFormatOptions);
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
if (null != dict) {
for (final Word w : dict) {
if (w.mIsBlacklistEntry) {
newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
} else {
newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
}
}
for (final Word w : dict) {
// By construction a binary dictionary may not have bigrams pointing to
// words that are not also registered as unigrams so we don't have to avoid
// them explicitly here.
for (final WeightedString bigram : w.mBigrams) {
newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
}
}
}
return newDict;
}
/**
* Helper method to pass a file name instead of a File object to isBinaryDictionary.
*/
public static boolean isBinaryDictionary(final String filename) {
final File file = new File(filename);
return isBinaryDictionary(file);
}
/**
* Basic test to find out whether the file is a binary dictionary or not.
*
* Concretely this only tests the magic number.
*
* @param file The file to test.
* @return true if it's a binary dictionary, false otherwise
*/
public static boolean isBinaryDictionary(final File file) {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
final int version = getFormatVersion(new ByteBufferWrapper(buffer));
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
} catch (FileNotFoundException e) {
return false;
} catch (IOException e) {
return false;
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
}
/**
* Calculate bigram frequency from compressed value
*
* @see #makeBigramFlags
*
* @param unigramFrequency
* @param bigramFrequency compressed frequency
* @return approximate bigram frequency
*/
public static int reconstructBigramFrequency(final int unigramFrequency,
final int bigramFrequency) {
final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
/ (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
return (int)resultFreqFloat;
}
} }

6
java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java
View file

@ -17,7 +17,7 @@
package com.android.inputmethod.latin.makedict; package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.utils.ByteArrayWrapper; import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File; import java.io.File;
@ -55,7 +55,7 @@ public class BinaryDictReader {
} }
} }
if (buffer != null) { if (buffer != null) {
return new BinaryDictInputOutput.ByteBufferWrapper(buffer); return new BinaryDictInputUtils.ByteBufferWrapper(buffer);
} }
return null; return null;
} }
@ -103,7 +103,7 @@ public class BinaryDictReader {
} }
} }
if (buffer != null) { if (buffer != null) {
return new BinaryDictInputOutput.ByteBufferWrapper(buffer); return new BinaryDictInputUtils.ByteBufferWrapper(buffer);
} }
return null; return null;
} }

30
java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
View file

@ -18,7 +18,7 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@ -58,7 +58,7 @@ public final class DynamicBinaryDictIOUtils {
public static void deleteWord(final FusionDictionaryBufferInterface buffer, public static void deleteWord(final FusionDictionaryBufferInterface buffer,
final String word) throws IOException, UnsupportedFormatException { final String word) throws IOException, UnsupportedFormatException {
buffer.position(0); buffer.position(0);
final FileHeader header = BinaryDictInputOutput.readHeader(buffer); final FileHeader header = BinaryDictInputUtils.readHeader(buffer);
final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word); final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word);
if (wordPosition == FormatSpec.NOT_VALID_WORD) return; if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
@ -114,7 +114,7 @@ public final class DynamicBinaryDictIOUtils {
final int originalPosition = buffer.position(); final int originalPosition = buffer.position();
buffer.position(nodeOriginAddress); buffer.position(nodeOriginAddress);
do { do {
final int count = BinaryDictInputOutput.readCharGroupCount(buffer); final int count = BinaryDictInputUtils.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) { for (int i = 0; i < count; ++i) {
updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions); updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions);
BinaryDictIOUtils.skipCharGroup(buffer, formatOptions); BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
@ -140,7 +140,7 @@ public final class DynamicBinaryDictIOUtils {
final int originalPosition = buffer.position(); final int originalPosition = buffer.position();
buffer.position(groupOriginAddress); buffer.position(groupOriginAddress);
final int flags = buffer.readUnsignedByte(); final int flags = buffer.readUnsignedByte();
final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions); final int parentAddress = BinaryDictInputUtils.readParentAddress(buffer, formatOptions);
BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS
@ -174,7 +174,7 @@ public final class DynamicBinaryDictIOUtils {
buffer.position(nodeOriginAddress); buffer.position(nodeOriginAddress);
int jumpCount = 0; int jumpCount = 0;
while (jumpCount++ < MAX_JUMPS) { while (jumpCount++ < MAX_JUMPS) {
final int count = BinaryDictInputOutput.readCharGroupCount(buffer); final int count = BinaryDictInputUtils.readCharGroupCount(buffer);
for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions); for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
final int forwardLinkAddress = buffer.readUnsignedInt24(); final int forwardLinkAddress = buffer.readUnsignedInt24();
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
@ -269,7 +269,7 @@ public final class DynamicBinaryDictIOUtils {
// find the insert position of the word. // find the insert position of the word.
if (buffer.position() != 0) buffer.position(0); if (buffer.position() != 0) buffer.position(0);
final FileHeader header = BinaryDictInputOutput.readHeader(buffer); final FileHeader header = BinaryDictInputUtils.readHeader(buffer);
int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position(); int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position();
final int[] codePoints = FusionDictionary.getCodePoints(word); final int[] codePoints = FusionDictionary.getCodePoints(word);
@ -279,12 +279,12 @@ public final class DynamicBinaryDictIOUtils {
if (wordPos >= wordLen) break; if (wordPos >= wordLen) break;
nodeOriginAddress = buffer.position(); nodeOriginAddress = buffer.position();
int nodeParentAddress = -1; int nodeParentAddress = -1;
final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); final int charGroupCount = BinaryDictInputUtils.readCharGroupCount(buffer);
boolean foundNextGroup = false; boolean foundNextGroup = false;
for (int i = 0; i < charGroupCount; ++i) { for (int i = 0; i < charGroupCount; ++i) {
address = buffer.position(); address = buffer.position();
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, final CharGroupInfo currentInfo = BinaryDictInputUtils.readCharGroup(buffer,
buffer.position(), header.mFormatOptions); buffer.position(), header.mFormatOptions);
final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags,
header.mFormatOptions); header.mFormatOptions);
@ -305,7 +305,7 @@ public final class DynamicBinaryDictIOUtils {
* abc - d - ef * abc - d - ef
*/ */
final int newNodeAddress = buffer.limit(); final int newNodeAddress = buffer.limit();
final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, final int flags = BinaryDictOutputUtils.makeCharGroupFlags(p > 1,
isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */,
false /* isBlackListEntry */, header.mFormatOptions); false /* isBlackListEntry */, header.mFormatOptions);
int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags,
@ -344,7 +344,7 @@ public final class DynamicBinaryDictIOUtils {
final int childrenAddress = currentInfo.mChildrenAddress; final int childrenAddress = currentInfo.mChildrenAddress;
// move prefix // move prefix
final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, final int prefixFlags = BinaryDictOutputUtils.makeCharGroupFlags(p > 1,
false /* isTerminal */, 0 /* childrenAddressSize*/, false /* isTerminal */, 0 /* childrenAddressSize*/,
false /* hasShortcut */, false /* hasBigrams */, false /* hasShortcut */, false /* hasBigrams */,
false /* isNotAWord */, false /* isBlackListEntry */, false /* isNotAWord */, false /* isBlackListEntry */,
@ -360,7 +360,7 @@ public final class DynamicBinaryDictIOUtils {
updateParentAddresses(buffer, currentInfo.mChildrenAddress, updateParentAddresses(buffer, currentInfo.mChildrenAddress,
newNodeAddress + written + 1, header.mFormatOptions); newNodeAddress + written + 1, header.mFormatOptions);
} }
final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags( final int suffixFlags = BinaryDictOutputUtils.makeCharGroupFlags(
suffixCharacters.length > 1, suffixCharacters.length > 1,
(currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0,
0 /* childrenAddressSize */, 0 /* childrenAddressSize */,
@ -378,7 +378,7 @@ public final class DynamicBinaryDictIOUtils {
final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p,
codePoints.length); codePoints.length);
final int flags = BinaryDictInputOutput.makeCharGroupFlags( final int flags = BinaryDictOutputUtils.makeCharGroupFlags(
newCharacters.length > 1, isTerminal, newCharacters.length > 1, isTerminal,
0 /* childrenAddressSize */, hasShortcuts, hasBigrams, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions); isNotAWord, isBlackListEntry, header.mFormatOptions);
@ -401,7 +401,7 @@ public final class DynamicBinaryDictIOUtils {
// only update group. // only update group.
final int newNodeAddress = buffer.limit(); final int newNodeAddress = buffer.limit();
final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; final boolean hasMultipleChars = currentInfo.mCharacters.length > 1;
final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, final int flags = BinaryDictOutputUtils.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions); isNotAWord, isBlackListEntry, header.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
@ -431,7 +431,7 @@ public final class DynamicBinaryDictIOUtils {
header.mFormatOptions); header.mFormatOptions);
final int newGroupAddress = newNodeAddress + 1; final int newGroupAddress = newNodeAddress + 1;
final boolean hasMultipleChars = (wordLen - wordPos) > 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1;
final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, final int flags = BinaryDictOutputUtils.makeCharGroupFlags(hasMultipleChars,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions); isNotAWord, isBlackListEntry, header.mFormatOptions);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
@ -476,7 +476,7 @@ public final class DynamicBinaryDictIOUtils {
BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress); BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress);
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1, final int flags = BinaryDictOutputUtils.makeCharGroupFlags(characters.length > 1,
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
isNotAWord, isBlackListEntry, header.mFormatOptions); isNotAWord, isBlackListEntry, header.mFormatOptions);
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,

2
java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java
View file

@ -16,7 +16,7 @@
package com.android.inputmethod.latin.utils; package com.android.inputmethod.latin.utils;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
/** /**
* This class provides an implementation for the FusionDictionary buffer interface that is backed * This class provides an implementation for the FusionDictionary buffer interface that is backed

7
java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
View file

@ -20,7 +20,8 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictOutputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -62,7 +63,7 @@ public final class UserHistoryDictIOUtils {
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);
try { try {
BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions); BinaryDictOutputUtils.writeDictionaryBinary(destination, fusionDict, formatOptions);
Log.d(TAG, "end writing"); Log.d(TAG, "end writing");
} catch (IOException e) { } catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e); Log.e(TAG, "IO exception while writing file", e);
@ -156,7 +157,7 @@ public final class UserHistoryDictIOUtils {
continue; continue;
} }
to.setBigram(word1, word2, to.setBigram(word1, word2,
BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency, BinaryDictInputUtils.reconstructBigramFrequency(unigramFrequency,
attr.mFrequency)); attr.mFrequency));
} }
} }

2
native/jni/src/suggest/core/dictionary/probability_utils.h
View file

@ -41,7 +41,7 @@ class ProbabilityUtils {
// the unigram probability to be the median value of the 17th step from the top. A value of // the unigram probability to be the median value of the 17th step from the top. A value of
// 0 for the bigram probability represents the middle of the 16th step from the top, // 0 for the bigram probability represents the middle of the 16th step from the top,
// while a value of 15 represents the middle of the top step. // while a value of 15 represents the middle of the top step.
// See makedict.BinaryDictInputOutput for details. // See makedict.BinaryDictInputUtils for details.
const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
/ (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
return unigramProbability return unigramProbability

14
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
View file

@ -22,7 +22,7 @@ import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log; import android.util.Log;
import android.util.SparseArray; import android.util.SparseArray;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node; import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@ -44,7 +44,7 @@ import java.util.Random;
import java.util.Set; import java.util.Set;
/** /**
* Unit tests for BinaryDictInputOutput * Unit tests for BinaryDictInputUtils and BinaryDictOutputUtils.
*/ */
@LargeTest @LargeTest
public class BinaryDictIOTests extends AndroidTestCase { public class BinaryDictIOTests extends AndroidTestCase {
@ -206,7 +206,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
// If you need to dump the dict to a textual file, uncomment the line below and the // If you need to dump the dict to a textual file, uncomment the line below and the
// function above // function above
// dumpToCombinedFileForDebug(file, "/tmp/foo"); // dumpToCombinedFileForDebug(file, "/tmp/foo");
BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions); BinaryDictOutputUtils.writeDictionaryBinary(out, dict, formatOptions);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
out.flush(); out.flush();
@ -272,7 +272,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
getBuffer(reader, bufferType); getBuffer(reader, bufferType);
assertNotNull(reader.getBuffer()); assertNotNull(reader.getBuffer());
now = System.currentTimeMillis(); now = System.currentTimeMillis();
dict = BinaryDictInputOutput.readDictionaryBinary(reader, null); dict = BinaryDictInputUtils.readDictionaryBinary(reader, null);
diff = System.currentTimeMillis() - now; diff = System.currentTimeMillis() - now;
} catch (IOException e) { } catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e); Log.e(TAG, "IOException while reading dictionary", e);
@ -383,7 +383,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
} }
actBigrams.get(word1).add(word2); actBigrams.get(word1).add(word2);
final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency( final int bigramFreq = BinaryDictInputUtils.reconstructBigramFrequency(
unigramFreq, attr.mFrequency); unigramFreq, attr.mFrequency);
assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ);
} }
@ -497,14 +497,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
FileHeader header = null; FileHeader header = null;
try { try {
header = BinaryDictInputOutput.readHeader(buffer); header = BinaryDictInputUtils.readHeader(buffer);
} catch (IOException e) { } catch (IOException e) {
return null; return null;
} catch (UnsupportedFormatException e) { } catch (UnsupportedFormatException e) {
return null; return null;
} }
if (header == null) return null; if (header == null) return null;
return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize, return BinaryDictInputUtils.getWordAtAddress(buffer, header.mHeaderSize,
address - header.mHeaderSize, header.mFormatOptions).mWord; address - header.mHeaderSize, header.mFormatOptions).mWord;
} }

20
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View file

@ -21,8 +21,8 @@ import android.test.MoreAsserts;
import android.test.suitebuilder.annotation.LargeTest; import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log; import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.ByteBufferWrapper; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.ByteBufferWrapper;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node; import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@ -114,10 +114,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private static void printNode(final FusionDictionaryBufferInterface buffer, private static void printNode(final FusionDictionaryBufferInterface buffer,
final FormatSpec.FormatOptions formatOptions) { final FormatSpec.FormatOptions formatOptions) {
Log.d(TAG, "Node at " + buffer.position()); Log.d(TAG, "Node at " + buffer.position());
final int count = BinaryDictInputOutput.readCharGroupCount(buffer); final int count = BinaryDictInputUtils.readCharGroupCount(buffer);
Log.d(TAG, " charGroupCount = " + count); Log.d(TAG, " charGroupCount = " + count);
for (int i = 0; i < count; ++i) { for (int i = 0; i < count; ++i) {
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, final CharGroupInfo currentInfo = BinaryDictInputUtils.readCharGroup(buffer,
buffer.position(), formatOptions); buffer.position(), formatOptions);
printCharGroup(currentInfo); printCharGroup(currentInfo);
} }
@ -129,7 +129,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private static void printBinaryFile(final FusionDictionaryBufferInterface buffer) private static void printBinaryFile(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException { throws IOException, UnsupportedFormatException {
FileHeader header = BinaryDictInputOutput.readHeader(buffer); FileHeader header = BinaryDictInputUtils.readHeader(buffer);
while (buffer.position() < buffer.limit()) { while (buffer.position() < buffer.limit()) {
printNode(buffer, header.mFormatOptions); printNode(buffer, header.mFormatOptions);
} }
@ -252,8 +252,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
inStream = new FileInputStream(file); inStream = new FileInputStream(file);
final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
final FileHeader header = BinaryDictInputOutput.readHeader(buffer); final FileHeader header = BinaryDictInputUtils.readHeader(buffer);
assertEquals(word, BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize, assertEquals(word, BinaryDictInputUtils.getWordAtAddress(buffer, header.mHeaderSize,
position - header.mHeaderSize, header.mFormatOptions).mWord); position - header.mHeaderSize, header.mFormatOptions).mWord);
} catch (IOException e) { } catch (IOException e) {
} catch (UnsupportedFormatException e) { } catch (UnsupportedFormatException e) {
@ -283,7 +283,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
try { try {
final FileOutputStream out = new FileOutputStream(file); final FileOutputStream out = new FileOutputStream(file);
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); BinaryDictOutputUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close(); out.close();
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
@ -335,7 +335,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
try { try {
final FileOutputStream out = new FileOutputStream(file); final FileOutputStream out = new FileOutputStream(file);
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); BinaryDictOutputUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close(); out.close();
} catch (IOException e) { } catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e); fail("IOException while writing an initial dictionary : " + e);
@ -372,7 +372,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
try { try {
final FileOutputStream out = new FileOutputStream(file); final FileOutputStream out = new FileOutputStream(file);
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); BinaryDictOutputUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
out.close(); out.close();
} catch (IOException e) { } catch (IOException e) {
assertTrue(false); assertTrue(false);

2
tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java
View file

@ -16,7 +16,7 @@
package com.android.inputmethod.latin.makedict; package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory; import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader. import com.android.inputmethod.latin.makedict.BinaryDictReader.
FusionDictionaryBufferFromByteArrayFactory; FusionDictionaryBufferFromByteArrayFactory;

6
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View file

@ -16,7 +16,7 @@
package com.android.inputmethod.latin.dicttool; package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
@ -99,7 +99,7 @@ public final class BinaryDictOffdeviceUtils {
// over and over, ending in a stack overflow. Hence we limit the depth at which we try // over and over, ending in a stack overflow. Hence we limit the depth at which we try
// decoding the file. // decoding the file.
if (depth > MAX_DECODE_DEPTH) return null; if (depth > MAX_DECODE_DEPTH) return null;
if (BinaryDictInputOutput.isBinaryDictionary(src)) { if (BinaryDictInputUtils.isBinaryDictionary(src)) {
spec.mFile = src; spec.mFile = src;
return spec; return spec;
} }
@ -194,7 +194,7 @@ public final class BinaryDictOffdeviceUtils {
System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
} }
return BinaryDictInputOutput.readDictionaryBinary(reader, null); return BinaryDictInputUtils.readDictionaryBinary(reader, null);
} }
} }
} catch (IOException e) { } catch (IOException e) {

11
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View file

@ -16,7 +16,8 @@
package com.android.inputmethod.latin.dicttool; package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictOutputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -177,7 +178,7 @@ public class DictionaryMaker {
inputUnigramXml = filename; inputUnigramXml = filename;
} else if (CombinedInputOutput.isCombinedDictionary(filename)) { } else if (CombinedInputOutput.isCombinedDictionary(filename)) {
inputCombined = filename; inputCombined = filename;
} else if (BinaryDictInputOutput.isBinaryDictionary(filename)) { } else if (BinaryDictInputUtils.isBinaryDictionary(filename)) {
inputBinary = filename; inputBinary = filename;
} else { } else {
throw new IllegalArgumentException( throw new IllegalArgumentException(
@ -199,7 +200,7 @@ public class DictionaryMaker {
} }
} else { } else {
if (null == inputBinary && null == inputUnigramXml) { if (null == inputBinary && null == inputUnigramXml) {
if (BinaryDictInputOutput.isBinaryDictionary(arg)) { if (BinaryDictInputUtils.isBinaryDictionary(arg)) {
inputBinary = arg; inputBinary = arg;
} else if (CombinedInputOutput.isCombinedDictionary(arg)) { } else if (CombinedInputOutput.isCombinedDictionary(arg)) {
inputCombined = arg; inputCombined = arg;
@ -269,7 +270,7 @@ public class DictionaryMaker {
final File file = new File(binaryFilename); final File file = new File(binaryFilename);
final BinaryDictReader reader = new BinaryDictReader(file); final BinaryDictReader reader = new BinaryDictReader(file);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
return BinaryDictInputOutput.readDictionaryBinary(reader, null); return BinaryDictInputUtils.readDictionaryBinary(reader, null);
} }
/** /**
@ -358,7 +359,7 @@ public class DictionaryMaker {
throws FileNotFoundException, IOException, UnsupportedFormatException { throws FileNotFoundException, IOException, UnsupportedFormatException {
final File outputFile = new File(outputFilename); final File outputFile = new File(outputFilename);
final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version);
BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, BinaryDictOutputUtils.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
formatOptions); formatOptions);
} }

7
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View file

@ -16,7 +16,8 @@
package com.android.inputmethod.latin.dicttool; package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictInputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictOutputUtils;
import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -61,7 +62,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
Compress.getCompressedStream( Compress.getCompressedStream(
new BufferedOutputStream(new FileOutputStream(dst))))); new BufferedOutputStream(new FileOutputStream(dst)))));
BinaryDictInputOutput.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); BinaryDictOutputUtils.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
// Test for an actually compressed dictionary and its contents // Test for an actually compressed dictionary and its contents
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec = final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
@ -72,7 +73,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile); final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader, final FusionDictionary resultDict = BinaryDictInputUtils.readDictionaryBinary(reader,
null /* dict : an optional dictionary to add words to, or null */); null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly", assertEquals("Dictionary can't be read back correctly",
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ); resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);

4
tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java
View file

@ -25,7 +25,7 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
/** /**
* Unit tests for BinaryDictInputOutput. * Unit tests for BinaryDictInputUtils and BinaryDictOutputUtils.
*/ */
public class BinaryDictInputOutputTest extends TestCase { public class BinaryDictInputOutputTest extends TestCase {
// Test the flattened array contains the expected number of nodes, and // Test the flattened array contains the expected number of nodes, and
@ -39,7 +39,7 @@ public class BinaryDictInputOutputTest extends TestCase {
dict.add("ftb", 1, null, false /* isNotAWord */); dict.add("ftb", 1, null, false /* isNotAWord */);
dict.add("bar", 1, null, false /* isNotAWord */); dict.add("bar", 1, null, false /* isNotAWord */);
dict.add("fool", 1, null, false /* isNotAWord */); dict.add("fool", 1, null, false /* isNotAWord */);
final ArrayList<Node> result = BinaryDictInputOutput.flattenTree(dict.mRoot); final ArrayList<Node> result = BinaryDictOutputUtils.flattenTree(dict.mRoot);
assertEquals(4, result.size()); assertEquals(4, result.size());
while (!result.isEmpty()) { while (!result.isEmpty()) {
final Node n = result.remove(0); final Node n = result.remove(0);

2
tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java
View file

@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.Random; import java.util.Random;
/** /**
* Unit tests for BinaryDictInputOutput. * Unit tests for BinaryDictInputUtils and BinaryDictOutputUtils.
*/ */
public class FusionDictionaryTest extends TestCase { public class FusionDictionaryTest extends TestCase {
private static final ArrayList<String> sWords = new ArrayList<String>(); private static final ArrayList<String> sWords = new ArrayList<String>();