Merge "[Refactor] Divide BinaryDictInputOutput into BinaryDictEncoder and BinaryDictDecoder."
Ken Wakasa
Android (Google) Code Review
commit
33fce975c9
|
|
@ -21,7 +21,7 @@ import android.content.SharedPreferences;
|
|||
import android.content.res.AssetFileDescriptor;
|
||||
import android.util.Log;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
|
||||
|
|
@ -231,8 +231,8 @@ final public class BinaryDictionaryGetter {
|
|||
try {
|
||||
// Read the version of the file
|
||||
inStream = new FileInputStream(f);
|
||||
final BinaryDictInputOutput.ByteBufferWrapper buffer =
|
||||
new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
|
||||
final BinaryDictDecoder.ByteBufferWrapper buffer =
|
||||
new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, f.length()));
|
||||
final int magic = buffer.readInt();
|
||||
if (magic != FormatSpec.MAGIC_NUMBER) {
|
||||
|
|
@ -241,7 +241,7 @@ final public class BinaryDictionaryGetter {
|
|||
final int formatVersion = buffer.readInt();
|
||||
final int headerSize = buffer.readInt();
|
||||
final HashMap<String, String> options = CollectionUtils.newHashMap();
|
||||
BinaryDictInputOutput.populateOptions(buffer, headerSize, options);
|
||||
BinaryDictDecoder.populateOptions(buffer, headerSize, options);
|
||||
|
||||
final String version = options.get(VERSION_KEY);
|
||||
if (null == version) {
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import android.content.Context;
|
|||
|
||||
import com.android.inputmethod.keyboard.ProximityInfo;
|
||||
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
|
|
@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter {
|
|||
@Override
|
||||
protected void writeBinaryDictionary(final FileOutputStream out)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,819 @@
|
|||
/*
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
import com.android.inputmethod.latin.utils.JniUtils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* Decodes binary files for a FusionDictionary.
|
||||
*
|
||||
* All the methods in this class are static.
|
||||
*/
|
||||
public final class BinaryDictDecoder {
|
||||
|
||||
private static final boolean DBG = MakedictLog.DBG;
|
||||
|
||||
static {
|
||||
JniUtils.loadNativeLibrary();
|
||||
}
|
||||
|
||||
// TODO: implement something sensical instead of just a phony method
|
||||
private static native int doNothing();
|
||||
|
||||
private BinaryDictDecoder() {
|
||||
// This utility class is not publicly instantiable.
|
||||
}
|
||||
|
||||
private static final int MAX_JUMPS = 12;
|
||||
|
||||
@UsedForTesting
|
||||
public interface FusionDictionaryBufferInterface {
|
||||
public int readUnsignedByte();
|
||||
public int readUnsignedShort();
|
||||
public int readUnsignedInt24();
|
||||
public int readInt();
|
||||
public int position();
|
||||
public void position(int newPosition);
|
||||
public void put(final byte b);
|
||||
public int limit();
|
||||
@UsedForTesting
|
||||
public int capacity();
|
||||
}
|
||||
|
||||
public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
|
||||
private ByteBuffer mBuffer;
|
||||
|
||||
public ByteBufferWrapper(final ByteBuffer buffer) {
|
||||
mBuffer = buffer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedByte() {
|
||||
return mBuffer.get() & 0xFF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedShort() {
|
||||
return mBuffer.getShort() & 0xFFFF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedInt24() {
|
||||
final int retval = readUnsignedByte();
|
||||
return (retval << 16) + readUnsignedShort();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readInt() {
|
||||
return mBuffer.getInt();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int position() {
|
||||
return mBuffer.position();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void position(int newPos) {
|
||||
mBuffer.position(newPos);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put(final byte b) {
|
||||
mBuffer.put(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int limit() {
|
||||
return mBuffer.limit();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int capacity() {
|
||||
return mBuffer.capacity();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A class grouping utility function for our specific character encoding.
|
||||
*/
|
||||
static final class CharEncoding {
|
||||
private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
|
||||
private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
|
||||
|
||||
/**
|
||||
* Helper method to find out whether this code fits on one byte
|
||||
*/
|
||||
private static boolean fitsOnOneByte(final int character) {
|
||||
return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
|
||||
&& character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the size of a character given its character code.
|
||||
*
|
||||
* Char format is:
|
||||
* 1 byte = bbbbbbbb match
|
||||
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
|
||||
* else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
|
||||
* unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
|
||||
* 00011111 would be outside unicode.
|
||||
* else: iso-latin-1 code
|
||||
* This allows for the whole unicode range to be encoded, including chars outside of
|
||||
* the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
|
||||
* characters which should never happen anyway (and still work, but take 3 bytes).
|
||||
*
|
||||
* @param character the character code.
|
||||
* @return the size in binary encoded-form, either 1 or 3 bytes.
|
||||
*/
|
||||
static int getCharSize(final int character) {
|
||||
// See char encoding in FusionDictionary.java
|
||||
if (fitsOnOneByte(character)) return 1;
|
||||
if (FormatSpec.INVALID_CHARACTER == character) return 1;
|
||||
return 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the byte size of a character array.
|
||||
*/
|
||||
static int getCharArraySize(final int[] chars) {
|
||||
int size = 0;
|
||||
for (int character : chars) size += getCharSize(character);
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a char array to a byte buffer.
|
||||
*
|
||||
* @param codePoints the code point array to write.
|
||||
* @param buffer the byte buffer to write to.
|
||||
* @param index the index in buffer to write the character array to.
|
||||
* @return the index after the last character.
|
||||
*/
|
||||
static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
|
||||
for (int codePoint : codePoints) {
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer[index++] = (byte)codePoint;
|
||||
} else {
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
|
||||
buffer[index++] = (byte)(0xFF & codePoint);
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a string with our character format to a byte buffer.
|
||||
*
|
||||
* This will also write the terminator byte.
|
||||
*
|
||||
* @param buffer the byte buffer to write to.
|
||||
* @param origin the offset to write from.
|
||||
* @param word the string to write.
|
||||
* @return the size written, in bytes.
|
||||
*/
|
||||
static int writeString(final byte[] buffer, final int origin,
|
||||
final String word) {
|
||||
final int length = word.length();
|
||||
int index = origin;
|
||||
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
|
||||
final int codePoint = word.codePointAt(i);
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer[index++] = (byte)codePoint;
|
||||
} else {
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
|
||||
buffer[index++] = (byte)(0xFF & codePoint);
|
||||
}
|
||||
}
|
||||
buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
|
||||
return index - origin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a string with our character format to a ByteArrayOutputStream.
|
||||
*
|
||||
* This will also write the terminator byte.
|
||||
*
|
||||
* @param buffer the ByteArrayOutputStream to write to.
|
||||
* @param word the string to write.
|
||||
*/
|
||||
static void writeString(final ByteArrayOutputStream buffer, final String word) {
|
||||
final int length = word.length();
|
||||
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
|
||||
final int codePoint = word.codePointAt(i);
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer.write((byte) codePoint);
|
||||
} else {
|
||||
buffer.write((byte) (0xFF & (codePoint >> 16)));
|
||||
buffer.write((byte) (0xFF & (codePoint >> 8)));
|
||||
buffer.write((byte) (0xFF & codePoint));
|
||||
}
|
||||
}
|
||||
buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a string from a buffer. This is the converse of the above method.
|
||||
*/
|
||||
private static String readString(final FusionDictionaryBufferInterface buffer) {
|
||||
final StringBuilder s = new StringBuilder();
|
||||
int character = readChar(buffer);
|
||||
while (character != FormatSpec.INVALID_CHARACTER) {
|
||||
s.appendCodePoint(character);
|
||||
character = readChar(buffer);
|
||||
}
|
||||
return s.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a character from the buffer.
|
||||
*
|
||||
* This follows the character format documented earlier in this source file.
|
||||
*
|
||||
* @param buffer the buffer, positioned over an encoded character.
|
||||
* @return the character code.
|
||||
*/
|
||||
static int readChar(final FusionDictionaryBufferInterface buffer) {
|
||||
int character = buffer.readUnsignedByte();
|
||||
if (!fitsOnOneByte(character)) {
|
||||
if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
|
||||
return FormatSpec.INVALID_CHARACTER;
|
||||
}
|
||||
character <<= 16;
|
||||
character += buffer.readUnsignedShort();
|
||||
}
|
||||
return character;
|
||||
}
|
||||
}
|
||||
|
||||
// Input methods: Read a binary dictionary to memory.
|
||||
// readDictionaryBinary is the public entry point for them.
|
||||
|
||||
static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
|
||||
final int optionFlags, final FormatOptions options) {
|
||||
if (options.mSupportsDynamicUpdate) {
|
||||
final int address = buffer.readUnsignedInt24();
|
||||
if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
|
||||
if ((address & FormatSpec.MSB24) != 0) {
|
||||
return -(address & FormatSpec.SINT24_MAX);
|
||||
} else {
|
||||
return address;
|
||||
}
|
||||
}
|
||||
int address;
|
||||
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
|
||||
return buffer.readUnsignedByte();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
|
||||
return buffer.readUnsignedShort();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
|
||||
return buffer.readUnsignedInt24();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
|
||||
default:
|
||||
return FormatSpec.NO_CHILDREN_ADDRESS;
|
||||
}
|
||||
}
|
||||
|
||||
static int readParentAddress(final FusionDictionaryBufferInterface buffer,
|
||||
final FormatOptions formatOptions) {
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
|
||||
final int parentAddress = buffer.readUnsignedInt24();
|
||||
final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
|
||||
return sign * (parentAddress & FormatSpec.SINT24_MAX);
|
||||
} else {
|
||||
return FormatSpec.NO_PARENT_ADDRESS;
|
||||
}
|
||||
}
|
||||
|
||||
private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
|
||||
public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
|
||||
final int originalGroupAddress, final FormatOptions options) {
|
||||
int addressPointer = originalGroupAddress;
|
||||
final int flags = buffer.readUnsignedByte();
|
||||
++addressPointer;
|
||||
|
||||
final int parentAddress = readParentAddress(buffer, options);
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
|
||||
addressPointer += 3;
|
||||
}
|
||||
|
||||
final int characters[];
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
|
||||
int index = 0;
|
||||
int character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
while (-1 != character) {
|
||||
// FusionDictionary is making sure that the length of the word is smaller than
|
||||
// MAX_WORD_LENGTH.
|
||||
// So we'll never write past the end of CHARACTER_BUFFER.
|
||||
CHARACTER_BUFFER[index++] = character;
|
||||
character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
}
|
||||
characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
|
||||
} else {
|
||||
final int character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
characters = new int[] { character };
|
||||
}
|
||||
final int frequency;
|
||||
if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
|
||||
++addressPointer;
|
||||
frequency = buffer.readUnsignedByte();
|
||||
} else {
|
||||
frequency = CharGroup.NOT_A_TERMINAL;
|
||||
}
|
||||
int childrenAddress = readChildrenAddress(buffer, flags, options);
|
||||
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
||||
childrenAddress += addressPointer;
|
||||
}
|
||||
addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
|
||||
ArrayList<WeightedString> shortcutTargets = null;
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
|
||||
final int pointerBefore = buffer.position();
|
||||
shortcutTargets = new ArrayList<WeightedString>();
|
||||
buffer.readUnsignedShort(); // Skip the size
|
||||
while (true) {
|
||||
final int targetFlags = buffer.readUnsignedByte();
|
||||
final String word = CharEncoding.readString(buffer);
|
||||
shortcutTargets.add(new WeightedString(word,
|
||||
targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
|
||||
if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
|
||||
}
|
||||
addressPointer += buffer.position() - pointerBefore;
|
||||
}
|
||||
ArrayList<PendingAttribute> bigrams = null;
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
|
||||
bigrams = new ArrayList<PendingAttribute>();
|
||||
int bigramCount = 0;
|
||||
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
|
||||
final int bigramFlags = buffer.readUnsignedByte();
|
||||
++addressPointer;
|
||||
final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
|
||||
? 1 : -1;
|
||||
int bigramAddress = addressPointer;
|
||||
switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
|
||||
bigramAddress += sign * buffer.readUnsignedByte();
|
||||
addressPointer += 1;
|
||||
break;
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
|
||||
bigramAddress += sign * buffer.readUnsignedShort();
|
||||
addressPointer += 2;
|
||||
break;
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
|
||||
final int offset = (buffer.readUnsignedByte() << 16)
|
||||
+ buffer.readUnsignedShort();
|
||||
bigramAddress += sign * offset;
|
||||
addressPointer += 3;
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Has bigrams with no address");
|
||||
}
|
||||
bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
|
||||
bigramAddress));
|
||||
if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
|
||||
}
|
||||
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
|
||||
MakedictLog.d("too many bigrams in a group.");
|
||||
}
|
||||
}
|
||||
return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
|
||||
parentAddress, childrenAddress, shortcutTargets, bigrams);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads and returns the char group count out of a buffer and forwards the pointer.
|
||||
*/
|
||||
public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
|
||||
final int msb = buffer.readUnsignedByte();
|
||||
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
|
||||
return msb;
|
||||
} else {
|
||||
return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
|
||||
+ buffer.readUnsignedByte();
|
||||
}
|
||||
}
|
||||
|
||||
// The word cache here is a stopgap bandaid to help the catastrophic performance
|
||||
// of this method. Since it performs direct, unbuffered random access to the file and
|
||||
// may be called hundreds of thousands of times, the resulting performance is not
|
||||
// reasonable without some kind of cache. Thus:
|
||||
private static TreeMap<Integer, WeightedString> wordCache =
|
||||
new TreeMap<Integer, WeightedString>();
|
||||
/**
|
||||
* Finds, as a string, the word at the address passed as an argument.
|
||||
*
|
||||
* @param buffer the buffer to read from.
|
||||
* @param headerSize the size of the header.
|
||||
* @param address the address to seek.
|
||||
* @param formatOptions file format options.
|
||||
* @return the word with its frequency, as a weighted string.
|
||||
*/
|
||||
/* package for tests */ static WeightedString getWordAtAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions formatOptions) {
|
||||
final WeightedString cachedString = wordCache.get(address);
|
||||
if (null != cachedString) return cachedString;
|
||||
|
||||
final WeightedString result;
|
||||
final int originalPointer = buffer.position();
|
||||
buffer.position(address);
|
||||
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
|
||||
result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
|
||||
} else {
|
||||
result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
|
||||
formatOptions);
|
||||
}
|
||||
|
||||
wordCache.put(address, result);
|
||||
buffer.position(originalPointer);
|
||||
return result;
|
||||
}
|
||||
|
||||
// TODO: static!? This will behave erratically when used in multi-threaded code.
|
||||
// We need to fix this
|
||||
private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
|
||||
@SuppressWarnings("unused")
|
||||
private static WeightedString getWordAtAddressWithParentAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions options) {
|
||||
int currentAddress = address;
|
||||
int index = FormatSpec.MAX_WORD_LENGTH - 1;
|
||||
int frequency = Integer.MIN_VALUE;
|
||||
// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
|
||||
for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
|
||||
CharGroupInfo currentInfo;
|
||||
int loopCounter = 0;
|
||||
do {
|
||||
buffer.position(currentAddress + headerSize);
|
||||
currentInfo = readCharGroup(buffer, currentAddress, options);
|
||||
if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
|
||||
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
|
||||
}
|
||||
if (DBG && loopCounter++ > MAX_JUMPS) {
|
||||
MakedictLog.d("Too many jumps - probably a bug");
|
||||
}
|
||||
} while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
|
||||
if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
|
||||
for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
|
||||
sGetWordBuffer[index--] =
|
||||
currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
|
||||
}
|
||||
if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
|
||||
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
|
||||
}
|
||||
|
||||
return new WeightedString(
|
||||
new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
|
||||
frequency);
|
||||
}
|
||||
|
||||
private static WeightedString getWordAtAddressWithoutParentAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions options) {
|
||||
buffer.position(headerSize);
|
||||
final int count = readCharGroupCount(buffer);
|
||||
int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
WeightedString result = null;
|
||||
|
||||
CharGroupInfo last = null;
|
||||
for (int i = count - 1; i >= 0; --i) {
|
||||
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
|
||||
groupOffset = info.mEndAddress;
|
||||
if (info.mOriginalAddress == address) {
|
||||
builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
|
||||
result = new WeightedString(builder.toString(), info.mFrequency);
|
||||
break; // and return
|
||||
}
|
||||
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
|
||||
if (info.mChildrenAddress > address) {
|
||||
if (null == last) continue;
|
||||
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
|
||||
buffer.position(last.mChildrenAddress + headerSize);
|
||||
i = readCharGroupCount(buffer);
|
||||
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
|
||||
last = null;
|
||||
continue;
|
||||
}
|
||||
last = info;
|
||||
}
|
||||
if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
|
||||
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
|
||||
buffer.position(last.mChildrenAddress + headerSize);
|
||||
i = readCharGroupCount(buffer);
|
||||
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
|
||||
last = null;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a single node from a buffer.
|
||||
*
|
||||
* This methods reads the file at the current position. A node is fully expected to start at
|
||||
* the current position.
|
||||
* This will recursively read other nodes into the structure, populating the reverse
|
||||
* maps on the fly and using them to keep track of already read nodes.
|
||||
*
|
||||
* @param buffer the buffer, correctly positioned at the start of a node.
|
||||
* @param headerSize the size, in bytes, of the file header.
|
||||
* @param reverseNodeMap a mapping from addresses to already read nodes.
|
||||
* @param reverseGroupMap a mapping from addresses to already read character groups.
|
||||
* @param options file format options.
|
||||
* @return the read node with all his children already read.
|
||||
*/
|
||||
private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
|
||||
final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
|
||||
final FormatOptions options)
|
||||
throws IOException {
|
||||
final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
|
||||
final int nodeOrigin = buffer.position() - headerSize;
|
||||
|
||||
do { // Scan the linked-list node.
|
||||
final int nodeHeadPosition = buffer.position() - headerSize;
|
||||
final int count = readCharGroupCount(buffer);
|
||||
int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
|
||||
for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
|
||||
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
|
||||
if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
|
||||
ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
|
||||
ArrayList<WeightedString> bigrams = null;
|
||||
if (null != info.mBigrams) {
|
||||
bigrams = new ArrayList<WeightedString>();
|
||||
for (PendingAttribute bigram : info.mBigrams) {
|
||||
final WeightedString word = getWordAtAddress(
|
||||
buffer, headerSize, bigram.mAddress, options);
|
||||
final int reconstructedFrequency =
|
||||
reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
|
||||
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
|
||||
}
|
||||
}
|
||||
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
|
||||
Node children = reverseNodeMap.get(info.mChildrenAddress);
|
||||
if (null == children) {
|
||||
final int currentPosition = buffer.position();
|
||||
buffer.position(info.mChildrenAddress + headerSize);
|
||||
children = readNode(
|
||||
buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
|
||||
buffer.position(currentPosition);
|
||||
}
|
||||
nodeContents.add(
|
||||
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
|
||||
info.mFrequency,
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
|
||||
} else {
|
||||
nodeContents.add(
|
||||
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
|
||||
info.mFrequency,
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
|
||||
}
|
||||
groupOffset = info.mEndAddress;
|
||||
}
|
||||
|
||||
// reach the end of the array.
|
||||
if (options.mSupportsDynamicUpdate) {
|
||||
final int nextAddress = buffer.readUnsignedInt24();
|
||||
if (nextAddress >= 0 && nextAddress < buffer.limit()) {
|
||||
buffer.position(nextAddress);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (options.mSupportsDynamicUpdate &&
|
||||
buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
|
||||
|
||||
final Node node = new Node(nodeContents);
|
||||
node.mCachedAddressBeforeUpdate = nodeOrigin;
|
||||
node.mCachedAddressAfterUpdate = nodeOrigin;
|
||||
reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get the binary format version from the header.
|
||||
* @throws IOException
|
||||
*/
|
||||
private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException {
|
||||
final int magic = buffer.readInt();
|
||||
if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
|
||||
return FormatSpec.NOT_A_VERSION_NUMBER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get and validate the binary format version.
|
||||
* @throws UnsupportedFormatException
|
||||
* @throws IOException
|
||||
*/
|
||||
private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
final int version = getFormatVersion(buffer);
|
||||
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|
||||
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
|
||||
throw new UnsupportedFormatException("This file has version " + version
|
||||
+ ", but this implementation does not support versions above "
|
||||
+ FormatSpec.MAXIMUM_SUPPORTED_VERSION);
|
||||
}
|
||||
return version;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a header from a buffer.
|
||||
* @param buffer the buffer to read.
|
||||
* @throws IOException
|
||||
* @throws UnsupportedFormatException
|
||||
*/
|
||||
public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
final int version = checkFormatVersion(buffer);
|
||||
final int optionsFlags = buffer.readUnsignedShort();
|
||||
|
||||
final HashMap<String, String> attributes = new HashMap<String, String>();
|
||||
final int headerSize;
|
||||
headerSize = buffer.readInt();
|
||||
|
||||
if (headerSize < 0) {
|
||||
throw new UnsupportedFormatException("header size can't be negative.");
|
||||
}
|
||||
|
||||
populateOptions(buffer, headerSize, attributes);
|
||||
buffer.position(headerSize);
|
||||
|
||||
final FileHeader header = new FileHeader(headerSize,
|
||||
new FusionDictionary.DictionaryOptions(attributes,
|
||||
0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
|
||||
0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
|
||||
new FormatOptions(version,
|
||||
0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
|
||||
return header;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads options from a buffer and populate a map with their contents.
|
||||
*
|
||||
* The buffer is read at the current position, so the caller must take care the pointer
|
||||
* is in the right place before calling this.
|
||||
*/
|
||||
public static void populateOptions(final FusionDictionaryBufferInterface buffer,
|
||||
final int headerSize, final HashMap<String, String> options) {
|
||||
while (buffer.position() < headerSize) {
|
||||
final String key = CharEncoding.readString(buffer);
|
||||
final String value = CharEncoding.readString(buffer);
|
||||
options.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a buffer and returns the memory representation of the dictionary.
|
||||
*
|
||||
* This high-level method takes a buffer and reads its contents, populating a
|
||||
* FusionDictionary structure. The optional dict argument is an existing dictionary to
|
||||
* which words from the buffer should be added. If it is null, a new dictionary is created.
|
||||
*
|
||||
* @param reader the reader.
|
||||
* @param dict an optional dictionary to add words to, or null.
|
||||
* @return the created (or merged) dictionary.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
|
||||
final FusionDictionary dict) throws FileNotFoundException, IOException,
|
||||
UnsupportedFormatException {
|
||||
// clear cache
|
||||
wordCache.clear();
|
||||
|
||||
// if the buffer has not been opened, open the buffer with bytebuffer.
|
||||
if (reader.getBuffer() == null) reader.openBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
if (reader.getBuffer() == null) {
|
||||
MakedictLog.e("Cannot open the buffer");
|
||||
}
|
||||
|
||||
// Read header
|
||||
final FileHeader header = readHeader(reader.getBuffer());
|
||||
|
||||
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
|
||||
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
|
||||
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
|
||||
reverseGroupMapping, header.mFormatOptions);
|
||||
|
||||
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
|
||||
if (null != dict) {
|
||||
for (final Word w : dict) {
|
||||
if (w.mIsBlacklistEntry) {
|
||||
newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
|
||||
} else {
|
||||
newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
|
||||
}
|
||||
}
|
||||
for (final Word w : dict) {
|
||||
// By construction a binary dictionary may not have bigrams pointing to
|
||||
// words that are not also registered as unigrams so we don't have to avoid
|
||||
// them explicitly here.
|
||||
for (final WeightedString bigram : w.mBigrams) {
|
||||
newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newDict;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to pass a file name instead of a File object to isBinaryDictionary.
|
||||
*/
|
||||
public static boolean isBinaryDictionary(final String filename) {
|
||||
final File file = new File(filename);
|
||||
return isBinaryDictionary(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic test to find out whether the file is a binary dictionary or not.
|
||||
*
|
||||
* Concretely this only tests the magic number.
|
||||
*
|
||||
* @param file The file to test.
|
||||
* @return true if it's a binary dictionary, false otherwise
|
||||
*/
|
||||
public static boolean isBinaryDictionary(final File file) {
|
||||
FileInputStream inStream = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||
final int version = getFormatVersion(new ByteBufferWrapper(buffer));
|
||||
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
|
||||
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
|
||||
} catch (FileNotFoundException e) {
|
||||
return false;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate bigram frequency from compressed value
|
||||
*
|
||||
* @see #makeBigramFlags
|
||||
*
|
||||
* @param unigramFrequency
|
||||
* @param bigramFrequency compressed frequency
|
||||
* @return approximate bigram frequency
|
||||
*/
|
||||
public static int reconstructBigramFrequency(final int unigramFrequency,
|
||||
final int bigramFrequency) {
|
||||
final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
|
||||
/ (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
|
||||
final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
|
||||
return (int)resultFreqFloat;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2011 The Android Open Source Project
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -16,275 +16,38 @@
|
|||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
import com.android.inputmethod.latin.utils.JniUtils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* Reads and writes XML files for a FusionDictionary.
|
||||
* Encodes binary files for a FusionDictionary.
|
||||
*
|
||||
* All the methods in this class are static.
|
||||
*/
|
||||
public final class BinaryDictInputOutput {
|
||||
public class BinaryDictEncoder {
|
||||
|
||||
private static final boolean DBG = MakedictLog.DBG;
|
||||
|
||||
static {
|
||||
JniUtils.loadNativeLibrary();
|
||||
private BinaryDictEncoder() {
|
||||
// This utility class is not publicly instantiable.
|
||||
}
|
||||
|
||||
// TODO: implement something sensical instead of just a phony method
|
||||
private static native int doNothing();
|
||||
|
||||
// Arbitrary limit to how much passes we consider address size compression should
|
||||
// terminate in. At the time of this writing, our largest dictionary completes
|
||||
// compression in five passes.
|
||||
// If the number of passes exceeds this number, makedict bails with an exception on
|
||||
// suspicion that a bug might be causing an infinite loop.
|
||||
private static final int MAX_PASSES = 24;
|
||||
private static final int MAX_JUMPS = 12;
|
||||
|
||||
@UsedForTesting
|
||||
public interface FusionDictionaryBufferInterface {
|
||||
public int readUnsignedByte();
|
||||
public int readUnsignedShort();
|
||||
public int readUnsignedInt24();
|
||||
public int readInt();
|
||||
public int position();
|
||||
public void position(int newPosition);
|
||||
public void put(final byte b);
|
||||
public int limit();
|
||||
@UsedForTesting
|
||||
public int capacity();
|
||||
}
|
||||
|
||||
public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
|
||||
private ByteBuffer mBuffer;
|
||||
|
||||
public ByteBufferWrapper(final ByteBuffer buffer) {
|
||||
mBuffer = buffer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedByte() {
|
||||
return mBuffer.get() & 0xFF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedShort() {
|
||||
return mBuffer.getShort() & 0xFFFF;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readUnsignedInt24() {
|
||||
final int retval = readUnsignedByte();
|
||||
return (retval << 16) + readUnsignedShort();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readInt() {
|
||||
return mBuffer.getInt();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int position() {
|
||||
return mBuffer.position();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void position(int newPos) {
|
||||
mBuffer.position(newPos);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put(final byte b) {
|
||||
mBuffer.put(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int limit() {
|
||||
return mBuffer.limit();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int capacity() {
|
||||
return mBuffer.capacity();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A class grouping utility function for our specific character encoding.
|
||||
*/
|
||||
static final class CharEncoding {
|
||||
private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20;
|
||||
private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF;
|
||||
|
||||
/**
|
||||
* Helper method to find out whether this code fits on one byte
|
||||
*/
|
||||
private static boolean fitsOnOneByte(final int character) {
|
||||
return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE
|
||||
&& character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the size of a character given its character code.
|
||||
*
|
||||
* Char format is:
|
||||
* 1 byte = bbbbbbbb match
|
||||
* case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte
|
||||
* else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because
|
||||
* unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with
|
||||
* 00011111 would be outside unicode.
|
||||
* else: iso-latin-1 code
|
||||
* This allows for the whole unicode range to be encoded, including chars outside of
|
||||
* the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control
|
||||
* characters which should never happen anyway (and still work, but take 3 bytes).
|
||||
*
|
||||
* @param character the character code.
|
||||
* @return the size in binary encoded-form, either 1 or 3 bytes.
|
||||
*/
|
||||
static int getCharSize(final int character) {
|
||||
// See char encoding in FusionDictionary.java
|
||||
if (fitsOnOneByte(character)) return 1;
|
||||
if (FormatSpec.INVALID_CHARACTER == character) return 1;
|
||||
return 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the byte size of a character array.
|
||||
*/
|
||||
private static int getCharArraySize(final int[] chars) {
|
||||
int size = 0;
|
||||
for (int character : chars) size += getCharSize(character);
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a char array to a byte buffer.
|
||||
*
|
||||
* @param codePoints the code point array to write.
|
||||
* @param buffer the byte buffer to write to.
|
||||
* @param index the index in buffer to write the character array to.
|
||||
* @return the index after the last character.
|
||||
*/
|
||||
private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) {
|
||||
for (int codePoint : codePoints) {
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer[index++] = (byte)codePoint;
|
||||
} else {
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
|
||||
buffer[index++] = (byte)(0xFF & codePoint);
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a string with our character format to a byte buffer.
|
||||
*
|
||||
* This will also write the terminator byte.
|
||||
*
|
||||
* @param buffer the byte buffer to write to.
|
||||
* @param origin the offset to write from.
|
||||
* @param word the string to write.
|
||||
* @return the size written, in bytes.
|
||||
*/
|
||||
private static int writeString(final byte[] buffer, final int origin,
|
||||
final String word) {
|
||||
final int length = word.length();
|
||||
int index = origin;
|
||||
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
|
||||
final int codePoint = word.codePointAt(i);
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer[index++] = (byte)codePoint;
|
||||
} else {
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 16));
|
||||
buffer[index++] = (byte)(0xFF & (codePoint >> 8));
|
||||
buffer[index++] = (byte)(0xFF & codePoint);
|
||||
}
|
||||
}
|
||||
buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR;
|
||||
return index - origin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a string with our character format to a ByteArrayOutputStream.
|
||||
*
|
||||
* This will also write the terminator byte.
|
||||
*
|
||||
* @param buffer the ByteArrayOutputStream to write to.
|
||||
* @param word the string to write.
|
||||
*/
|
||||
private static void writeString(final ByteArrayOutputStream buffer, final String word) {
|
||||
final int length = word.length();
|
||||
for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
|
||||
final int codePoint = word.codePointAt(i);
|
||||
if (1 == getCharSize(codePoint)) {
|
||||
buffer.write((byte) codePoint);
|
||||
} else {
|
||||
buffer.write((byte) (0xFF & (codePoint >> 16)));
|
||||
buffer.write((byte) (0xFF & (codePoint >> 8)));
|
||||
buffer.write((byte) (0xFF & codePoint));
|
||||
}
|
||||
}
|
||||
buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a string from a buffer. This is the converse of the above method.
|
||||
*/
|
||||
private static String readString(final FusionDictionaryBufferInterface buffer) {
|
||||
final StringBuilder s = new StringBuilder();
|
||||
int character = readChar(buffer);
|
||||
while (character != FormatSpec.INVALID_CHARACTER) {
|
||||
s.appendCodePoint(character);
|
||||
character = readChar(buffer);
|
||||
}
|
||||
return s.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a character from the buffer.
|
||||
*
|
||||
* This follows the character format documented earlier in this source file.
|
||||
*
|
||||
* @param buffer the buffer, positioned over an encoded character.
|
||||
* @return the character code.
|
||||
*/
|
||||
static int readChar(final FusionDictionaryBufferInterface buffer) {
|
||||
int character = buffer.readUnsignedByte();
|
||||
if (!fitsOnOneByte(character)) {
|
||||
if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) {
|
||||
return FormatSpec.INVALID_CHARACTER;
|
||||
}
|
||||
character <<= 16;
|
||||
character += buffer.readUnsignedShort();
|
||||
}
|
||||
return character;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the binary size of the character array.
|
||||
|
|
@ -432,7 +195,7 @@ public final class BinaryDictInputOutput {
|
|||
}
|
||||
}
|
||||
|
||||
// End utility methods.
|
||||
// End utility methods
|
||||
|
||||
// This method is responsible for finding a nice ordering of the nodes that favors run-time
|
||||
// cache performance and dictionary size.
|
||||
|
|
@ -1227,557 +990,4 @@ public final class BinaryDictInputOutput {
|
|||
destination.close();
|
||||
MakedictLog.i("Done");
|
||||
}
|
||||
|
||||
|
||||
// Input methods: Read a binary dictionary to memory.
|
||||
// readDictionaryBinary is the public entry point for them.
|
||||
|
||||
static int getChildrenAddressSize(final int optionFlags,
|
||||
final FormatOptions formatOptions) {
|
||||
if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
|
||||
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
|
||||
return 1;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
|
||||
return 2;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
|
||||
return 3;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int readChildrenAddress(final FusionDictionaryBufferInterface buffer,
|
||||
final int optionFlags, final FormatOptions options) {
|
||||
if (options.mSupportsDynamicUpdate) {
|
||||
final int address = buffer.readUnsignedInt24();
|
||||
if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS;
|
||||
if ((address & FormatSpec.MSB24) != 0) {
|
||||
return -(address & FormatSpec.SINT24_MAX);
|
||||
} else {
|
||||
return address;
|
||||
}
|
||||
}
|
||||
int address;
|
||||
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
|
||||
return buffer.readUnsignedByte();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
|
||||
return buffer.readUnsignedShort();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
|
||||
return buffer.readUnsignedInt24();
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
|
||||
default:
|
||||
return FormatSpec.NO_CHILDREN_ADDRESS;
|
||||
}
|
||||
}
|
||||
|
||||
static int readParentAddress(final FusionDictionaryBufferInterface buffer,
|
||||
final FormatOptions formatOptions) {
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
|
||||
final int parentAddress = buffer.readUnsignedInt24();
|
||||
final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1;
|
||||
return sign * (parentAddress & FormatSpec.SINT24_MAX);
|
||||
} else {
|
||||
return FormatSpec.NO_PARENT_ADDRESS;
|
||||
}
|
||||
}
|
||||
|
||||
private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
|
||||
public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
|
||||
final int originalGroupAddress, final FormatOptions options) {
|
||||
int addressPointer = originalGroupAddress;
|
||||
final int flags = buffer.readUnsignedByte();
|
||||
++addressPointer;
|
||||
|
||||
final int parentAddress = readParentAddress(buffer, options);
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
|
||||
addressPointer += 3;
|
||||
}
|
||||
|
||||
final int characters[];
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
|
||||
int index = 0;
|
||||
int character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
while (-1 != character) {
|
||||
// FusionDictionary is making sure that the length of the word is smaller than
|
||||
// MAX_WORD_LENGTH.
|
||||
// So we'll never write past the end of CHARACTER_BUFFER.
|
||||
CHARACTER_BUFFER[index++] = character;
|
||||
character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
}
|
||||
characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index);
|
||||
} else {
|
||||
final int character = CharEncoding.readChar(buffer);
|
||||
addressPointer += CharEncoding.getCharSize(character);
|
||||
characters = new int[] { character };
|
||||
}
|
||||
final int frequency;
|
||||
if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
|
||||
++addressPointer;
|
||||
frequency = buffer.readUnsignedByte();
|
||||
} else {
|
||||
frequency = CharGroup.NOT_A_TERMINAL;
|
||||
}
|
||||
int childrenAddress = readChildrenAddress(buffer, flags, options);
|
||||
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
||||
childrenAddress += addressPointer;
|
||||
}
|
||||
addressPointer += getChildrenAddressSize(flags, options);
|
||||
ArrayList<WeightedString> shortcutTargets = null;
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
|
||||
final int pointerBefore = buffer.position();
|
||||
shortcutTargets = new ArrayList<WeightedString>();
|
||||
buffer.readUnsignedShort(); // Skip the size
|
||||
while (true) {
|
||||
final int targetFlags = buffer.readUnsignedByte();
|
||||
final String word = CharEncoding.readString(buffer);
|
||||
shortcutTargets.add(new WeightedString(word,
|
||||
targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY));
|
||||
if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
|
||||
}
|
||||
addressPointer += buffer.position() - pointerBefore;
|
||||
}
|
||||
ArrayList<PendingAttribute> bigrams = null;
|
||||
if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) {
|
||||
bigrams = new ArrayList<PendingAttribute>();
|
||||
int bigramCount = 0;
|
||||
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
|
||||
final int bigramFlags = buffer.readUnsignedByte();
|
||||
++addressPointer;
|
||||
final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE)
|
||||
? 1 : -1;
|
||||
int bigramAddress = addressPointer;
|
||||
switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
|
||||
bigramAddress += sign * buffer.readUnsignedByte();
|
||||
addressPointer += 1;
|
||||
break;
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
|
||||
bigramAddress += sign * buffer.readUnsignedShort();
|
||||
addressPointer += 2;
|
||||
break;
|
||||
case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
|
||||
final int offset = (buffer.readUnsignedByte() << 16)
|
||||
+ buffer.readUnsignedShort();
|
||||
bigramAddress += sign * offset;
|
||||
addressPointer += 3;
|
||||
break;
|
||||
default:
|
||||
throw new RuntimeException("Has bigrams with no address");
|
||||
}
|
||||
bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY,
|
||||
bigramAddress));
|
||||
if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break;
|
||||
}
|
||||
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) {
|
||||
MakedictLog.d("too many bigrams in a group.");
|
||||
}
|
||||
}
|
||||
return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency,
|
||||
parentAddress, childrenAddress, shortcutTargets, bigrams);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads and returns the char group count out of a buffer and forwards the pointer.
|
||||
*/
|
||||
public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
|
||||
final int msb = buffer.readUnsignedByte();
|
||||
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
|
||||
return msb;
|
||||
} else {
|
||||
return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
|
||||
+ buffer.readUnsignedByte();
|
||||
}
|
||||
}
|
||||
|
||||
// The word cache here is a stopgap bandaid to help the catastrophic performance
|
||||
// of this method. Since it performs direct, unbuffered random access to the file and
|
||||
// may be called hundreds of thousands of times, the resulting performance is not
|
||||
// reasonable without some kind of cache. Thus:
|
||||
private static TreeMap<Integer, WeightedString> wordCache =
|
||||
new TreeMap<Integer, WeightedString>();
|
||||
/**
|
||||
* Finds, as a string, the word at the address passed as an argument.
|
||||
*
|
||||
* @param buffer the buffer to read from.
|
||||
* @param headerSize the size of the header.
|
||||
* @param address the address to seek.
|
||||
* @param formatOptions file format options.
|
||||
* @return the word with its frequency, as a weighted string.
|
||||
*/
|
||||
/* package for tests */ static WeightedString getWordAtAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions formatOptions) {
|
||||
final WeightedString cachedString = wordCache.get(address);
|
||||
if (null != cachedString) return cachedString;
|
||||
|
||||
final WeightedString result;
|
||||
final int originalPointer = buffer.position();
|
||||
buffer.position(address);
|
||||
|
||||
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
|
||||
result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions);
|
||||
} else {
|
||||
result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address,
|
||||
formatOptions);
|
||||
}
|
||||
|
||||
wordCache.put(address, result);
|
||||
buffer.position(originalPointer);
|
||||
return result;
|
||||
}
|
||||
|
||||
// TODO: static!? This will behave erratically when used in multi-threaded code.
|
||||
// We need to fix this
|
||||
private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
|
||||
@SuppressWarnings("unused")
|
||||
private static WeightedString getWordAtAddressWithParentAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions options) {
|
||||
int currentAddress = address;
|
||||
int index = FormatSpec.MAX_WORD_LENGTH - 1;
|
||||
int frequency = Integer.MIN_VALUE;
|
||||
// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
|
||||
for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
|
||||
CharGroupInfo currentInfo;
|
||||
int loopCounter = 0;
|
||||
do {
|
||||
buffer.position(currentAddress + headerSize);
|
||||
currentInfo = readCharGroup(buffer, currentAddress, options);
|
||||
if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) {
|
||||
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
|
||||
}
|
||||
if (DBG && loopCounter++ > MAX_JUMPS) {
|
||||
MakedictLog.d("Too many jumps - probably a bug");
|
||||
}
|
||||
} while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options));
|
||||
if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
|
||||
for (int i = 0; i < currentInfo.mCharacters.length; ++i) {
|
||||
sGetWordBuffer[index--] =
|
||||
currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1];
|
||||
}
|
||||
if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
|
||||
currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
|
||||
}
|
||||
|
||||
return new WeightedString(
|
||||
new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1),
|
||||
frequency);
|
||||
}
|
||||
|
||||
private static WeightedString getWordAtAddressWithoutParentAddress(
|
||||
final FusionDictionaryBufferInterface buffer, final int headerSize, final int address,
|
||||
final FormatOptions options) {
|
||||
buffer.position(headerSize);
|
||||
final int count = readCharGroupCount(buffer);
|
||||
int groupOffset = BinaryDictIOUtils.getGroupCountSize(count);
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
WeightedString result = null;
|
||||
|
||||
CharGroupInfo last = null;
|
||||
for (int i = count - 1; i >= 0; --i) {
|
||||
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
|
||||
groupOffset = info.mEndAddress;
|
||||
if (info.mOriginalAddress == address) {
|
||||
builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
|
||||
result = new WeightedString(builder.toString(), info.mFrequency);
|
||||
break; // and return
|
||||
}
|
||||
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
|
||||
if (info.mChildrenAddress > address) {
|
||||
if (null == last) continue;
|
||||
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
|
||||
buffer.position(last.mChildrenAddress + headerSize);
|
||||
i = readCharGroupCount(buffer);
|
||||
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
|
||||
last = null;
|
||||
continue;
|
||||
}
|
||||
last = info;
|
||||
}
|
||||
if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) {
|
||||
builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
|
||||
buffer.position(last.mChildrenAddress + headerSize);
|
||||
i = readCharGroupCount(buffer);
|
||||
groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i);
|
||||
last = null;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a single node from a buffer.
|
||||
*
|
||||
* This methods reads the file at the current position. A node is fully expected to start at
|
||||
* the current position.
|
||||
* This will recursively read other nodes into the structure, populating the reverse
|
||||
* maps on the fly and using them to keep track of already read nodes.
|
||||
*
|
||||
* @param buffer the buffer, correctly positioned at the start of a node.
|
||||
* @param headerSize the size, in bytes, of the file header.
|
||||
* @param reverseNodeMap a mapping from addresses to already read nodes.
|
||||
* @param reverseGroupMap a mapping from addresses to already read character groups.
|
||||
* @param options file format options.
|
||||
* @return the read node with all his children already read.
|
||||
*/
|
||||
private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
|
||||
final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap,
|
||||
final FormatOptions options)
|
||||
throws IOException {
|
||||
final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>();
|
||||
final int nodeOrigin = buffer.position() - headerSize;
|
||||
|
||||
do { // Scan the linked-list node.
|
||||
final int nodeHeadPosition = buffer.position() - headerSize;
|
||||
final int count = readCharGroupCount(buffer);
|
||||
int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count);
|
||||
for (int i = count; i > 0; --i) { // Scan the array of CharGroup.
|
||||
CharGroupInfo info = readCharGroup(buffer, groupOffset, options);
|
||||
if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue;
|
||||
ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
|
||||
ArrayList<WeightedString> bigrams = null;
|
||||
if (null != info.mBigrams) {
|
||||
bigrams = new ArrayList<WeightedString>();
|
||||
for (PendingAttribute bigram : info.mBigrams) {
|
||||
final WeightedString word = getWordAtAddress(
|
||||
buffer, headerSize, bigram.mAddress, options);
|
||||
final int reconstructedFrequency =
|
||||
reconstructBigramFrequency(word.mFrequency, bigram.mFrequency);
|
||||
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
|
||||
}
|
||||
}
|
||||
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
|
||||
Node children = reverseNodeMap.get(info.mChildrenAddress);
|
||||
if (null == children) {
|
||||
final int currentPosition = buffer.position();
|
||||
buffer.position(info.mChildrenAddress + headerSize);
|
||||
children = readNode(
|
||||
buffer, headerSize, reverseNodeMap, reverseGroupMap, options);
|
||||
buffer.position(currentPosition);
|
||||
}
|
||||
nodeContents.add(
|
||||
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
|
||||
info.mFrequency,
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
|
||||
} else {
|
||||
nodeContents.add(
|
||||
new CharGroup(info.mCharacters, shortcutTargets, bigrams,
|
||||
info.mFrequency,
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
|
||||
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
|
||||
}
|
||||
groupOffset = info.mEndAddress;
|
||||
}
|
||||
|
||||
// reach the end of the array.
|
||||
if (options.mSupportsDynamicUpdate) {
|
||||
final int nextAddress = buffer.readUnsignedInt24();
|
||||
if (nextAddress >= 0 && nextAddress < buffer.limit()) {
|
||||
buffer.position(nextAddress);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (options.mSupportsDynamicUpdate &&
|
||||
buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
|
||||
|
||||
final Node node = new Node(nodeContents);
|
||||
node.mCachedAddressBeforeUpdate = nodeOrigin;
|
||||
node.mCachedAddressAfterUpdate = nodeOrigin;
|
||||
reverseNodeMap.put(node.mCachedAddressAfterUpdate, node);
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get the binary format version from the header.
|
||||
* @throws IOException
|
||||
*/
|
||||
private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException {
|
||||
final int magic = buffer.readInt();
|
||||
if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort();
|
||||
return FormatSpec.NOT_A_VERSION_NUMBER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get and validate the binary format version.
|
||||
* @throws UnsupportedFormatException
|
||||
* @throws IOException
|
||||
*/
|
||||
private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
final int version = getFormatVersion(buffer);
|
||||
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|
||||
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
|
||||
throw new UnsupportedFormatException("This file has version " + version
|
||||
+ ", but this implementation does not support versions above "
|
||||
+ FormatSpec.MAXIMUM_SUPPORTED_VERSION);
|
||||
}
|
||||
return version;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a header from a buffer.
|
||||
* @param buffer the buffer to read.
|
||||
* @throws IOException
|
||||
* @throws UnsupportedFormatException
|
||||
*/
|
||||
public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
final int version = checkFormatVersion(buffer);
|
||||
final int optionsFlags = buffer.readUnsignedShort();
|
||||
|
||||
final HashMap<String, String> attributes = new HashMap<String, String>();
|
||||
final int headerSize;
|
||||
headerSize = buffer.readInt();
|
||||
|
||||
if (headerSize < 0) {
|
||||
throw new UnsupportedFormatException("header size can't be negative.");
|
||||
}
|
||||
|
||||
populateOptions(buffer, headerSize, attributes);
|
||||
buffer.position(headerSize);
|
||||
|
||||
final FileHeader header = new FileHeader(headerSize,
|
||||
new FusionDictionary.DictionaryOptions(attributes,
|
||||
0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG),
|
||||
0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)),
|
||||
new FormatOptions(version,
|
||||
0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE)));
|
||||
return header;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads options from a buffer and populate a map with their contents.
|
||||
*
|
||||
* The buffer is read at the current position, so the caller must take care the pointer
|
||||
* is in the right place before calling this.
|
||||
*/
|
||||
public static void populateOptions(final FusionDictionaryBufferInterface buffer,
|
||||
final int headerSize, final HashMap<String, String> options) {
|
||||
while (buffer.position() < headerSize) {
|
||||
final String key = CharEncoding.readString(buffer);
|
||||
final String value = CharEncoding.readString(buffer);
|
||||
options.put(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a buffer and returns the memory representation of the dictionary.
|
||||
*
|
||||
* This high-level method takes a buffer and reads its contents, populating a
|
||||
* FusionDictionary structure. The optional dict argument is an existing dictionary to
|
||||
* which words from the buffer should be added. If it is null, a new dictionary is created.
|
||||
*
|
||||
* @param reader the reader.
|
||||
* @param dict an optional dictionary to add words to, or null.
|
||||
* @return the created (or merged) dictionary.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
|
||||
final FusionDictionary dict) throws FileNotFoundException, IOException,
|
||||
UnsupportedFormatException {
|
||||
// clear cache
|
||||
wordCache.clear();
|
||||
|
||||
// if the buffer has not been opened, open the buffer with bytebuffer.
|
||||
if (reader.getBuffer() == null) reader.openBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
if (reader.getBuffer() == null) {
|
||||
MakedictLog.e("Cannot open the buffer");
|
||||
}
|
||||
|
||||
// Read header
|
||||
final FileHeader header = readHeader(reader.getBuffer());
|
||||
|
||||
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
|
||||
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
|
||||
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
|
||||
reverseGroupMapping, header.mFormatOptions);
|
||||
|
||||
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
|
||||
if (null != dict) {
|
||||
for (final Word w : dict) {
|
||||
if (w.mIsBlacklistEntry) {
|
||||
newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord);
|
||||
} else {
|
||||
newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord);
|
||||
}
|
||||
}
|
||||
for (final Word w : dict) {
|
||||
// By construction a binary dictionary may not have bigrams pointing to
|
||||
// words that are not also registered as unigrams so we don't have to avoid
|
||||
// them explicitly here.
|
||||
for (final WeightedString bigram : w.mBigrams) {
|
||||
newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return newDict;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to pass a file name instead of a File object to isBinaryDictionary.
|
||||
*/
|
||||
public static boolean isBinaryDictionary(final String filename) {
|
||||
final File file = new File(filename);
|
||||
return isBinaryDictionary(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Basic test to find out whether the file is a binary dictionary or not.
|
||||
*
|
||||
* Concretely this only tests the magic number.
|
||||
*
|
||||
* @param file The file to test.
|
||||
* @return true if it's a binary dictionary, false otherwise
|
||||
*/
|
||||
public static boolean isBinaryDictionary(final File file) {
|
||||
FileInputStream inStream = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||
final int version = getFormatVersion(new ByteBufferWrapper(buffer));
|
||||
return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION
|
||||
&& version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION);
|
||||
} catch (FileNotFoundException e) {
|
||||
return false;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate bigram frequency from compressed value
|
||||
*
|
||||
* @see #makeBigramFlags
|
||||
*
|
||||
* @param unigramFrequency
|
||||
* @param bigramFrequency compressed frequency
|
||||
* @return approximate bigram frequency
|
||||
*/
|
||||
public static int reconstructBigramFrequency(final int unigramFrequency,
|
||||
final int bigramFrequency) {
|
||||
final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)
|
||||
/ (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY);
|
||||
final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f);
|
||||
return (int)resultFreqFloat;
|
||||
}
|
||||
}
|
||||
|
|
@ -18,8 +18,8 @@ package com.android.inputmethod.latin.makedict;
|
|||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.Constants;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
|
|
@ -86,7 +86,7 @@ public final class BinaryDictIOUtils {
|
|||
if (index != p.mLength) index = p.mLength;
|
||||
|
||||
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
|
||||
p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
p.mNumOfCharGroup = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
|
||||
p.mPosition = 0;
|
||||
}
|
||||
|
|
@ -94,7 +94,7 @@ public final class BinaryDictIOUtils {
|
|||
stack.pop();
|
||||
continue;
|
||||
}
|
||||
CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer,
|
||||
CharGroupInfo info = BinaryDictDecoder.readCharGroup(buffer,
|
||||
p.mAddress - headerSize, formatOptions);
|
||||
for (int i = 0; i < info.mCharacters.length; ++i) {
|
||||
pushedChars[index++] = info.mCharacters[i];
|
||||
|
|
@ -153,7 +153,7 @@ public final class BinaryDictIOUtils {
|
|||
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
|
||||
UnsupportedFormatException {
|
||||
// Read header
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(reader.getBuffer());
|
||||
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
|
||||
frequencies, bigrams, header.mFormatOptions);
|
||||
}
|
||||
|
|
@ -174,18 +174,18 @@ public final class BinaryDictIOUtils {
|
|||
if (word == null) return FormatSpec.NOT_VALID_WORD;
|
||||
if (buffer.position() != 0) buffer.position(0);
|
||||
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
int wordPos = 0;
|
||||
final int wordLen = word.codePointCount(0, word.length());
|
||||
for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) {
|
||||
if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD;
|
||||
|
||||
do {
|
||||
final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
boolean foundNextCharGroup = false;
|
||||
for (int i = 0; i < charGroupCount; ++i) {
|
||||
final int charGroupPos = buffer.position();
|
||||
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
|
||||
final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
|
||||
buffer.position(), header.mFormatOptions);
|
||||
final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags,
|
||||
header.mFormatOptions);
|
||||
|
|
@ -271,7 +271,7 @@ public final class BinaryDictIOUtils {
|
|||
*/
|
||||
private static int writeVariableAddress(final OutputStream destination, final int value)
|
||||
throws IOException {
|
||||
switch (BinaryDictInputOutput.getByteSize(value)) {
|
||||
switch (BinaryDictEncoder.getByteSize(value)) {
|
||||
case 1:
|
||||
destination.write((byte)value);
|
||||
break;
|
||||
|
|
@ -285,15 +285,15 @@ public final class BinaryDictIOUtils {
|
|||
destination.write((byte)(0xFF & value));
|
||||
break;
|
||||
}
|
||||
return BinaryDictInputOutput.getByteSize(value);
|
||||
return BinaryDictEncoder.getByteSize(value);
|
||||
}
|
||||
|
||||
static void skipCharGroup(final FusionDictionaryBufferInterface buffer,
|
||||
final FormatOptions formatOptions) {
|
||||
final int flags = buffer.readUnsignedByte();
|
||||
BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
|
||||
BinaryDictDecoder.readParentAddress(buffer, formatOptions);
|
||||
skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
|
||||
BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions);
|
||||
BinaryDictDecoder.readChildrenAddress(buffer, flags, formatOptions);
|
||||
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
|
||||
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
|
||||
final int shortcutsSize = buffer.readUnsignedShort();
|
||||
|
|
@ -411,14 +411,14 @@ public final class BinaryDictIOUtils {
|
|||
|
||||
if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) {
|
||||
final int shortcutListSize =
|
||||
BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
|
||||
BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
|
||||
destination.write((byte)(shortcutListSize >> 8));
|
||||
destination.write((byte)(shortcutListSize & 0xFF));
|
||||
size += 2;
|
||||
final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator();
|
||||
while (shortcutIterator.hasNext()) {
|
||||
final WeightedString target = shortcutIterator.next();
|
||||
destination.write((byte)BinaryDictInputOutput.makeShortcutFlags(
|
||||
destination.write((byte)BinaryDictEncoder.makeShortcutFlags(
|
||||
shortcutIterator.hasNext(), target.mFrequency));
|
||||
size++;
|
||||
size += writeString(destination, target.mWord);
|
||||
|
|
@ -427,7 +427,7 @@ public final class BinaryDictIOUtils {
|
|||
|
||||
if (info.mBigrams != null) {
|
||||
// TODO: Consolidate this code with the code that computes the size of the bigram list
|
||||
// in BinaryDictionaryInputOutput#computeActualNodeSize
|
||||
// in BinaryDictEncoder#computeActualNodeSize
|
||||
for (int i = 0; i < info.mBigrams.size(); ++i) {
|
||||
|
||||
final int bigramFrequency = info.mBigrams.get(i).mFrequency;
|
||||
|
|
@ -437,7 +437,7 @@ public final class BinaryDictIOUtils {
|
|||
final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress
|
||||
+ size);
|
||||
bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0;
|
||||
switch (BinaryDictInputOutput.getByteSize(bigramOffset)) {
|
||||
switch (BinaryDictEncoder.getByteSize(bigramOffset)) {
|
||||
case 1:
|
||||
bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
|
||||
break;
|
||||
|
|
@ -461,18 +461,18 @@ public final class BinaryDictIOUtils {
|
|||
*/
|
||||
static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) {
|
||||
int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
|
||||
+ BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters)
|
||||
+ BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions);
|
||||
+ BinaryDictEncoder.getGroupCharactersSize(info.mCharacters)
|
||||
+ getChildrenAddressSize(info.mFlags, formatOptions);
|
||||
if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) {
|
||||
size += FormatSpec.GROUP_FREQUENCY_SIZE;
|
||||
}
|
||||
if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) {
|
||||
size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets);
|
||||
size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets);
|
||||
}
|
||||
if (info.mBigrams != null) {
|
||||
for (final PendingAttribute attr : info.mBigrams) {
|
||||
size += FormatSpec.GROUP_FLAGS_SIZE;
|
||||
size += BinaryDictInputOutput.getByteSize(attr.mAddress);
|
||||
size += BinaryDictEncoder.getByteSize(attr.mAddress);
|
||||
}
|
||||
}
|
||||
return size;
|
||||
|
|
@ -520,9 +520,9 @@ public final class BinaryDictIOUtils {
|
|||
int position = getTerminalPosition(buffer, word);
|
||||
if (position != FormatSpec.NOT_VALID_WORD) {
|
||||
buffer.position(0);
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
buffer.position(position);
|
||||
return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions);
|
||||
return BinaryDictDecoder.readCharGroup(buffer, position, header.mFormatOptions);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -544,10 +544,10 @@ public final class BinaryDictIOUtils {
|
|||
final FileInputStream inStream = new FileInputStream(file);
|
||||
try {
|
||||
inStream.read(buffer);
|
||||
final BinaryDictInputOutput.ByteBufferWrapper wrapper =
|
||||
new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map(
|
||||
final BinaryDictDecoder.ByteBufferWrapper wrapper =
|
||||
new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, offset, length));
|
||||
return BinaryDictInputOutput.readHeader(wrapper);
|
||||
return BinaryDictDecoder.readHeader(wrapper);
|
||||
} finally {
|
||||
inStream.close();
|
||||
}
|
||||
|
|
@ -612,4 +612,20 @@ public final class BinaryDictIOUtils {
|
|||
+ ")");
|
||||
}
|
||||
}
|
||||
|
||||
static int getChildrenAddressSize(final int optionFlags,
|
||||
final FormatOptions formatOptions) {
|
||||
if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
|
||||
switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) {
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
|
||||
return 1;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
|
||||
return 2;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
|
||||
return 3;
|
||||
case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
|
||||
|
||||
import java.io.File;
|
||||
|
|
@ -55,7 +55,7 @@ public class BinaryDictReader {
|
|||
}
|
||||
}
|
||||
if (buffer != null) {
|
||||
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
|
||||
return new BinaryDictDecoder.ByteBufferWrapper(buffer);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -103,7 +103,7 @@ public class BinaryDictReader {
|
|||
}
|
||||
}
|
||||
if (buffer != null) {
|
||||
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
|
||||
return new BinaryDictDecoder.ByteBufferWrapper(buffer);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ package com.android.inputmethod.latin.makedict;
|
|||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.Constants;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
|
|
@ -58,7 +58,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
public static void deleteWord(final FusionDictionaryBufferInterface buffer,
|
||||
final String word) throws IOException, UnsupportedFormatException {
|
||||
buffer.position(0);
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word);
|
||||
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
|
||||
|
||||
|
|
@ -114,7 +114,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
final int originalPosition = buffer.position();
|
||||
buffer.position(nodeOriginAddress);
|
||||
do {
|
||||
final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
final int count = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions);
|
||||
BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
|
||||
|
|
@ -140,7 +140,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
final int originalPosition = buffer.position();
|
||||
buffer.position(groupOriginAddress);
|
||||
final int flags = buffer.readUnsignedByte();
|
||||
final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions);
|
||||
final int parentAddress = BinaryDictDecoder.readParentAddress(buffer, formatOptions);
|
||||
BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
|
||||
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte();
|
||||
final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS
|
||||
|
|
@ -174,7 +174,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
buffer.position(nodeOriginAddress);
|
||||
int jumpCount = 0;
|
||||
while (jumpCount++ < MAX_JUMPS) {
|
||||
final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
final int count = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions);
|
||||
final int forwardLinkAddress = buffer.readUnsignedInt24();
|
||||
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
|
||||
|
|
@ -269,7 +269,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
|
||||
// find the insert position of the word.
|
||||
if (buffer.position() != 0) buffer.position(0);
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
|
||||
int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position();
|
||||
final int[] codePoints = FusionDictionary.getCodePoints(word);
|
||||
|
|
@ -279,12 +279,12 @@ public final class DynamicBinaryDictIOUtils {
|
|||
if (wordPos >= wordLen) break;
|
||||
nodeOriginAddress = buffer.position();
|
||||
int nodeParentAddress = -1;
|
||||
final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
boolean foundNextGroup = false;
|
||||
|
||||
for (int i = 0; i < charGroupCount; ++i) {
|
||||
address = buffer.position();
|
||||
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
|
||||
final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
|
||||
buffer.position(), header.mFormatOptions);
|
||||
final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags,
|
||||
header.mFormatOptions);
|
||||
|
|
@ -305,7 +305,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
* abc - d - ef
|
||||
*/
|
||||
final int newNodeAddress = buffer.limit();
|
||||
final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
|
||||
final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
|
||||
isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */,
|
||||
false /* isBlackListEntry */, header.mFormatOptions);
|
||||
int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags,
|
||||
|
|
@ -344,7 +344,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
final int childrenAddress = currentInfo.mChildrenAddress;
|
||||
|
||||
// move prefix
|
||||
final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1,
|
||||
final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1,
|
||||
false /* isTerminal */, 0 /* childrenAddressSize*/,
|
||||
false /* hasShortcut */, false /* hasBigrams */,
|
||||
false /* isNotAWord */, false /* isBlackListEntry */,
|
||||
|
|
@ -360,7 +360,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
updateParentAddresses(buffer, currentInfo.mChildrenAddress,
|
||||
newNodeAddress + written + 1, header.mFormatOptions);
|
||||
}
|
||||
final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags(
|
||||
final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags(
|
||||
suffixCharacters.length > 1,
|
||||
(currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0,
|
||||
0 /* childrenAddressSize */,
|
||||
|
|
@ -378,7 +378,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
|
||||
final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p,
|
||||
codePoints.length);
|
||||
final int flags = BinaryDictInputOutput.makeCharGroupFlags(
|
||||
final int flags = BinaryDictEncoder.makeCharGroupFlags(
|
||||
newCharacters.length > 1, isTerminal,
|
||||
0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
|
||||
isNotAWord, isBlackListEntry, header.mFormatOptions);
|
||||
|
|
@ -401,7 +401,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
// only update group.
|
||||
final int newNodeAddress = buffer.limit();
|
||||
final boolean hasMultipleChars = currentInfo.mCharacters.length > 1;
|
||||
final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
|
||||
final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
|
||||
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
|
||||
isNotAWord, isBlackListEntry, header.mFormatOptions);
|
||||
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
|
||||
|
|
@ -431,7 +431,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
header.mFormatOptions);
|
||||
final int newGroupAddress = newNodeAddress + 1;
|
||||
final boolean hasMultipleChars = (wordLen - wordPos) > 1;
|
||||
final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars,
|
||||
final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars,
|
||||
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
|
||||
isNotAWord, isBlackListEntry, header.mFormatOptions);
|
||||
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
|
||||
|
|
@ -476,7 +476,7 @@ public final class DynamicBinaryDictIOUtils {
|
|||
BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress);
|
||||
|
||||
final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen);
|
||||
final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1,
|
||||
final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1,
|
||||
isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams,
|
||||
isNotAWord, isBlackListEntry, header.mFormatOptions);
|
||||
final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
package com.android.inputmethod.latin.utils;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
|
||||
/**
|
||||
* This class provides an implementation for the FusionDictionary buffer interface that is backed
|
||||
|
|
|
|||
|
|
@ -19,8 +19,9 @@ package com.android.inputmethod.latin.utils;
|
|||
import android.util.Log;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
|
|
@ -62,7 +63,7 @@ public final class UserHistoryDictIOUtils {
|
|||
final FormatOptions formatOptions) {
|
||||
final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);
|
||||
try {
|
||||
BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions);
|
||||
BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions);
|
||||
Log.d(TAG, "end writing");
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IO exception while writing file", e);
|
||||
|
|
@ -156,7 +157,7 @@ public final class UserHistoryDictIOUtils {
|
|||
continue;
|
||||
}
|
||||
to.setBigram(word1, word2,
|
||||
BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency,
|
||||
BinaryDictDecoder.reconstructBigramFrequency(unigramFrequency,
|
||||
attr.mFrequency));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ LATIN_IME_JNI_SRC_FILES := \
|
|||
com_android_inputmethod_keyboard_ProximityInfo.cpp \
|
||||
com_android_inputmethod_latin_BinaryDictionary.cpp \
|
||||
com_android_inputmethod_latin_DicTraverseSession.cpp \
|
||||
com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
|
||||
com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
|
||||
jni_common.cpp
|
||||
|
||||
LATIN_IME_CORE_SRC_FILES := \
|
||||
|
|
|
|||
|
|
@ -14,16 +14,16 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "LatinIME: jni: BinaryDictInputOutput"
|
||||
#define LOG_TAG "LatinIME: jni: BinaryDictDecoder"
|
||||
|
||||
#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
|
||||
#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
|
||||
|
||||
#include "defines.h"
|
||||
#include "jni.h"
|
||||
#include "jni_common.h"
|
||||
|
||||
namespace latinime {
|
||||
static int latinime_BinaryDictInputOutput_doNothing(JNIEnv *env, jclass clazz) {
|
||||
static int latinime_BinaryDictDecoder_doNothing(JNIEnv *env, jclass clazz) {
|
||||
// This is a phony method for test - it does nothing. It just returns some value
|
||||
// unlikely to be in memory by chance for testing purposes.
|
||||
// TODO: remove this method.
|
||||
|
|
@ -35,13 +35,13 @@ static const JNINativeMethod sMethods[] = {
|
|||
// TODO: remove this entry when we have one useful method in here
|
||||
const_cast<char *>("doNothing"),
|
||||
const_cast<char *>("()I"),
|
||||
reinterpret_cast<void *>(latinime_BinaryDictInputOutput_doNothing)
|
||||
reinterpret_cast<void *>(latinime_BinaryDictDecoder_doNothing)
|
||||
},
|
||||
};
|
||||
|
||||
int register_BinaryDictInputOutput(JNIEnv *env) {
|
||||
int register_BinaryDictDecoder(JNIEnv *env) {
|
||||
const char *const kClassPathName =
|
||||
"com/android/inputmethod/latin/makedict/BinaryDictInputOutput";
|
||||
"com/android/inputmethod/latin/makedict/BinaryDictDecoder";
|
||||
return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods));
|
||||
}
|
||||
} // namespace latinime
|
||||
|
|
@ -14,12 +14,12 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
|
||||
#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
|
||||
#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
|
||||
#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
|
||||
|
||||
#include "jni.h"
|
||||
|
||||
namespace latinime {
|
||||
int register_BinaryDictInputOutput(JNIEnv *env);
|
||||
int register_BinaryDictDecoder(JNIEnv *env);
|
||||
} // namespace latinime
|
||||
#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H
|
||||
#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
#include "com_android_inputmethod_latin_BinaryDictionary.h"
|
||||
#include "com_android_inputmethod_latin_DicTraverseSession.h"
|
||||
#endif
|
||||
#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h"
|
||||
#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h"
|
||||
#include "defines.h"
|
||||
|
||||
/*
|
||||
|
|
@ -55,8 +55,8 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) {
|
|||
return -1;
|
||||
}
|
||||
#endif
|
||||
if (!latinime::register_BinaryDictInputOutput(env)) {
|
||||
AKLOGE("ERROR: BinaryDictInputOutput native registration failed");
|
||||
if (!latinime::register_BinaryDictDecoder(env)) {
|
||||
AKLOGE("ERROR: BinaryDictDecoder native registration failed");
|
||||
return -1;
|
||||
}
|
||||
/* success -- return valid version number */
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class ProbabilityUtils {
|
|||
// the unigram probability to be the median value of the 17th step from the top. A value of
|
||||
// 0 for the bigram probability represents the middle of the 16th step from the top,
|
||||
// while a value of 15 represents the middle of the top step.
|
||||
// See makedict.BinaryDictInputOutput for details.
|
||||
// See makedict.BinaryDictDecoder for details.
|
||||
const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
|
||||
/ (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
|
||||
return unigramProbability
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import android.test.suitebuilder.annotation.LargeTest;
|
|||
import android.util.Log;
|
||||
import android.util.SparseArray;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
|
|
@ -44,11 +44,11 @@ import java.util.Random;
|
|||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Unit tests for BinaryDictInputOutput
|
||||
* Unit tests for BinaryDictDecoder and BinaryDictEncoder.
|
||||
*/
|
||||
@LargeTest
|
||||
public class BinaryDictIOTests extends AndroidTestCase {
|
||||
private static final String TAG = BinaryDictIOTests.class.getSimpleName();
|
||||
public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
||||
private static final String TAG = BinaryDictDecoderEncoderTests.class.getSimpleName();
|
||||
private static final int DEFAULT_MAX_UNIGRAMS = 100;
|
||||
private static final int DEFAULT_CODE_POINT_SET_SIZE = 50;
|
||||
private static final int UNIGRAM_FREQ = 10;
|
||||
|
|
@ -71,11 +71,11 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
|
||||
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
|
||||
|
||||
public BinaryDictIOTests() {
|
||||
public BinaryDictDecoderEncoderTests() {
|
||||
this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS);
|
||||
}
|
||||
|
||||
public BinaryDictIOTests(final long seed, final int maxUnigrams) {
|
||||
public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) {
|
||||
super();
|
||||
Log.e(TAG, "Testing dictionary: seed is " + seed);
|
||||
final Random random = new Random(seed);
|
||||
|
|
@ -206,7 +206,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
// If you need to dump the dict to a textual file, uncomment the line below and the
|
||||
// function above
|
||||
// dumpToCombinedFileForDebug(file, "/tmp/foo");
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions);
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
|
||||
out.flush();
|
||||
|
|
@ -272,7 +272,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
getBuffer(reader, bufferType);
|
||||
assertNotNull(reader.getBuffer());
|
||||
now = System.currentTimeMillis();
|
||||
dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
dict = BinaryDictDecoder.readDictionaryBinary(reader, null);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while reading dictionary", e);
|
||||
|
|
@ -383,7 +383,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
}
|
||||
actBigrams.get(word1).add(word2);
|
||||
|
||||
final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency(
|
||||
final int bigramFreq = BinaryDictDecoder.reconstructBigramFrequency(
|
||||
unigramFreq, attr.mFrequency);
|
||||
assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ);
|
||||
}
|
||||
|
|
@ -497,14 +497,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
|
||||
FileHeader header = null;
|
||||
try {
|
||||
header = BinaryDictInputOutput.readHeader(buffer);
|
||||
header = BinaryDictDecoder.readHeader(buffer);
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
} catch (UnsupportedFormatException e) {
|
||||
return null;
|
||||
}
|
||||
if (header == null) return null;
|
||||
return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
|
||||
return BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize,
|
||||
address - header.mHeaderSize, header.mFormatOptions).mWord;
|
||||
}
|
||||
|
||||
|
|
@ -21,8 +21,8 @@ import android.test.MoreAsserts;
|
|||
import android.test.suitebuilder.annotation.LargeTest;
|
||||
import android.util.Log;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.ByteBufferWrapper;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.ByteBufferWrapper;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
|
|
@ -114,10 +114,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
private static void printNode(final FusionDictionaryBufferInterface buffer,
|
||||
final FormatSpec.FormatOptions formatOptions) {
|
||||
Log.d(TAG, "Node at " + buffer.position());
|
||||
final int count = BinaryDictInputOutput.readCharGroupCount(buffer);
|
||||
final int count = BinaryDictDecoder.readCharGroupCount(buffer);
|
||||
Log.d(TAG, " charGroupCount = " + count);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer,
|
||||
final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer,
|
||||
buffer.position(), formatOptions);
|
||||
printCharGroup(currentInfo);
|
||||
}
|
||||
|
|
@ -129,7 +129,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
|
||||
private static void printBinaryFile(final FusionDictionaryBufferInterface buffer)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
while (buffer.position() < buffer.limit()) {
|
||||
printNode(buffer, header.mFormatOptions);
|
||||
}
|
||||
|
|
@ -252,8 +252,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
inStream = new FileInputStream(file);
|
||||
final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper(
|
||||
inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length()));
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
assertEquals(word, BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize,
|
||||
final FileHeader header = BinaryDictDecoder.readHeader(buffer);
|
||||
assertEquals(word, BinaryDictDecoder.getWordAtAddress(buffer, header.mHeaderSize,
|
||||
position - header.mHeaderSize, header.mFormatOptions).mWord);
|
||||
} catch (IOException e) {
|
||||
} catch (UnsupportedFormatException e) {
|
||||
|
|
@ -283,7 +283,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
|
||||
try {
|
||||
final FileOutputStream out = new FileOutputStream(file);
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
out.close();
|
||||
} catch (IOException e) {
|
||||
fail("IOException while writing an initial dictionary : " + e);
|
||||
|
|
@ -335,7 +335,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
|
||||
try {
|
||||
final FileOutputStream out = new FileOutputStream(file);
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
out.close();
|
||||
} catch (IOException e) {
|
||||
fail("IOException while writing an initial dictionary : " + e);
|
||||
|
|
@ -372,7 +372,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
|||
|
||||
try {
|
||||
final FileOutputStream out = new FileOutputStream(file);
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS);
|
||||
out.close();
|
||||
} catch (IOException e) {
|
||||
assertTrue(false);
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader.
|
||||
FusionDictionaryBufferFromByteArrayFactory;
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR)
|
|||
# Used in jni_common.cpp to avoid registering useless methods.
|
||||
|
||||
LATIN_IME_JNI_SRC_FILES := \
|
||||
com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \
|
||||
com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \
|
||||
jni_common.cpp
|
||||
|
||||
LATIN_IME_CORE_SRC_FILES :=
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
|
|
@ -99,7 +99,7 @@ public final class BinaryDictOffdeviceUtils {
|
|||
// over and over, ending in a stack overflow. Hence we limit the depth at which we try
|
||||
// decoding the file.
|
||||
if (depth > MAX_DECODE_DEPTH) return null;
|
||||
if (BinaryDictInputOutput.isBinaryDictionary(src)) {
|
||||
if (BinaryDictDecoder.isBinaryDictionary(src)) {
|
||||
spec.mFile = src;
|
||||
return spec;
|
||||
}
|
||||
|
|
@ -194,7 +194,7 @@ public final class BinaryDictOffdeviceUtils {
|
|||
System.out.println("Packaging : " + decodedSpec.describeChain());
|
||||
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
|
||||
}
|
||||
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
return BinaryDictDecoder.readDictionaryBinary(reader, null);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@
|
|||
|
||||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
|
|
@ -177,7 +178,7 @@ public class DictionaryMaker {
|
|||
inputUnigramXml = filename;
|
||||
} else if (CombinedInputOutput.isCombinedDictionary(filename)) {
|
||||
inputCombined = filename;
|
||||
} else if (BinaryDictInputOutput.isBinaryDictionary(filename)) {
|
||||
} else if (BinaryDictDecoder.isBinaryDictionary(filename)) {
|
||||
inputBinary = filename;
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
|
|
@ -199,7 +200,7 @@ public class DictionaryMaker {
|
|||
}
|
||||
} else {
|
||||
if (null == inputBinary && null == inputUnigramXml) {
|
||||
if (BinaryDictInputOutput.isBinaryDictionary(arg)) {
|
||||
if (BinaryDictDecoder.isBinaryDictionary(arg)) {
|
||||
inputBinary = arg;
|
||||
} else if (CombinedInputOutput.isCombinedDictionary(arg)) {
|
||||
inputCombined = arg;
|
||||
|
|
@ -269,7 +270,7 @@ public class DictionaryMaker {
|
|||
final File file = new File(binaryFilename);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
return BinaryDictDecoder.readDictionaryBinary(reader, null);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -358,7 +359,7 @@ public class DictionaryMaker {
|
|||
throws FileNotFoundException, IOException, UnsupportedFormatException {
|
||||
final File outputFile = new File(outputFilename);
|
||||
final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version);
|
||||
BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
|
||||
BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict,
|
||||
formatOptions);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@
|
|||
|
||||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictIOTests;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionaryTest;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
|
|
@ -37,9 +37,9 @@ public class Test extends Dicttool.Command {
|
|||
private static final Class<?>[] sClassesToTest = {
|
||||
BinaryDictOffdeviceUtilsTests.class,
|
||||
FusionDictionaryTest.class,
|
||||
BinaryDictInputOutputTest.class,
|
||||
BinaryDictDecoderEncoderTests.class
|
||||
BinaryDictEncoderFlattenTreeTests.class,
|
||||
BinaryDictIOUtilsTests.class,
|
||||
BinaryDictIOTests.class
|
||||
};
|
||||
private ArrayList<Method> mAllTestMethods = new ArrayList<Method>();
|
||||
private ArrayList<String> mUsedTestMethods = new ArrayList<String>();
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@
|
|||
|
||||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictDecoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictEncoder;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
|
|
@ -38,7 +39,7 @@ import java.util.ArrayList;
|
|||
import java.util.HashMap;
|
||||
|
||||
/**
|
||||
* Unit tests for BinaryDictOffdeviceUtilsTests
|
||||
* Unit tests for BinaryDictOffdeviceUtils
|
||||
*/
|
||||
public class BinaryDictOffdeviceUtilsTests extends TestCase {
|
||||
private static final int TEST_FREQ = 37; // Some arbitrary value unlikely to happen by chance
|
||||
|
|
@ -61,7 +62,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
|
|||
Compress.getCompressedStream(
|
||||
new BufferedOutputStream(new FileOutputStream(dst)))));
|
||||
|
||||
BinaryDictInputOutput.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
|
||||
BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false));
|
||||
|
||||
// Test for an actually compressed dictionary and its contents
|
||||
final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec =
|
||||
|
|
@ -72,7 +73,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
|
|||
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
|
||||
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
|
||||
final FusionDictionary resultDict = BinaryDictDecoder.readDictionaryBinary(reader,
|
||||
null /* dict : an optional dictionary to add words to, or null */);
|
||||
assertEquals("Dictionary can't be read back correctly",
|
||||
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);
|
||||
|
|
|
|||
|
|
@ -25,9 +25,9 @@ import java.util.ArrayList;
|
|||
import java.util.HashMap;
|
||||
|
||||
/**
|
||||
* Unit tests for BinaryDictInputOutput.
|
||||
* Unit tests for BinaryDictEncoder.flattenTree().
|
||||
*/
|
||||
public class BinaryDictInputOutputTest extends TestCase {
|
||||
public class BinaryDictEncoderFlattenTreeTests extends TestCase {
|
||||
// Test the flattened array contains the expected number of nodes, and
|
||||
// that it does not contain any duplicates.
|
||||
public void testFlattenNodes() {
|
||||
|
|
@ -39,7 +39,7 @@ public class BinaryDictInputOutputTest extends TestCase {
|
|||
dict.add("ftb", 1, null, false /* isNotAWord */);
|
||||
dict.add("bar", 1, null, false /* isNotAWord */);
|
||||
dict.add("fool", 1, null, false /* isNotAWord */);
|
||||
final ArrayList<Node> result = BinaryDictInputOutput.flattenTree(dict.mRoot);
|
||||
final ArrayList<Node> result = BinaryDictEncoder.flattenTree(dict.mRoot);
|
||||
assertEquals(4, result.size());
|
||||
while (!result.isEmpty()) {
|
||||
final Node n = result.remove(0);
|
||||
|
|
@ -29,7 +29,7 @@ import java.util.HashMap;
|
|||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* Unit tests for BinaryDictInputOutput.
|
||||
* Unit tests for FusionDictionary.
|
||||
*/
|
||||
public class FusionDictionaryTest extends TestCase {
|
||||
private static final ArrayList<String> sWords = new ArrayList<String>();
|
||||
|
|
|
|||
|
|
@ -24,5 +24,5 @@ fi
|
|||
find out -name "dicttool_aosp*" -exec rm -rf {} \; > /dev/null 2>&1
|
||||
mmm -j8 external/junit
|
||||
DICTTOOL_UNITTEST=true mmm -j8 packages/inputmethods/LatinIME/tools/dicttool
|
||||
java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest
|
||||
java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests
|
||||
java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.dicttool.BinaryDictOffdeviceUtilsTests
|
||||
|
|
|
|||
Loading…
Reference in New Issue