nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Add FusionDictionaryBufferInterface." into jb-mr1-dev

main
Ken Wakasa 2012-08-29 18:49:02 -07:00 committed by Android (Google) Code Review
parent 85cb45c80f f5c4ff4817
commit d2a81ab9bd
1 changed files with 107 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
View File

@ -188,6 +188,54 @@ public class BinaryDictInputOutput {
// suspicion that a bug might be causing an infinite loop. // suspicion that a bug might be causing an infinite loop.
private static final int MAX_PASSES = 24; private static final int MAX_PASSES = 24;
private interface FusionDictionaryBufferInterface {
public int readUnsignedByte();
public int readUnsignedShort();
public int readUnsignedInt24();
public int readInt();
public int position();
public void position(int newPosition);
}
private static final class ByteBufferWrapper implements FusionDictionaryBufferInterface {
private ByteBuffer buffer;
ByteBufferWrapper(final ByteBuffer buffer) {
this.buffer = buffer;
}
@Override
public int readUnsignedByte() {
return ((int)buffer.get()) & 0xFF;
}
@Override
public int readUnsignedShort() {
return ((int)buffer.getShort()) & 0xFFFF;
}
@Override
public int readUnsignedInt24() {
final int retval = readUnsignedByte();
return (retval << 16) + readUnsignedShort();
}
@Override
public int readInt() {
return buffer.getInt();
}
@Override
public int position() {
return buffer.position();
}
@Override
public void position(int newPos) {
buffer.position(newPos);
return;
}
}
/** /**
* A class grouping utility function for our specific character encoding. * A class grouping utility function for our specific character encoding.
*/ */
@ -310,9 +358,9 @@ public class BinaryDictInputOutput {
} }
/** /**
* Reads a string from a ByteBuffer. This is the converse of the above method. * Reads a string from a buffer. This is the converse of the above method.
*/ */
private static String readString(final ByteBuffer buffer) { private static String readString(final FusionDictionaryBufferInterface buffer) {
final StringBuilder s = new StringBuilder(); final StringBuilder s = new StringBuilder();
int character = readChar(buffer); int character = readChar(buffer);
while (character != INVALID_CHARACTER) { while (character != INVALID_CHARACTER) {
@ -323,19 +371,19 @@ public class BinaryDictInputOutput {
} }
/** /**
* Reads a character from the ByteBuffer. * Reads a character from the buffer.
* *
* This follows the character format documented earlier in this source file. * This follows the character format documented earlier in this source file.
* *
* @param buffer the buffer, positioned over an encoded character. * @param buffer the buffer, positioned over an encoded character.
* @return the character code. * @return the character code.
*/ */
private static int readChar(final ByteBuffer buffer) { private static int readChar(final FusionDictionaryBufferInterface buffer) {
int character = readUnsignedByte(buffer); int character = buffer.readUnsignedByte();
if (!fitsOnOneByte(character)) { if (!fitsOnOneByte(character)) {
if (GROUP_CHARACTERS_TERMINATOR == character) return INVALID_CHARACTER; if (GROUP_CHARACTERS_TERMINATOR == character) return INVALID_CHARACTER;
character <<= 16; character <<= 16;
character += readUnsignedShort(buffer); character += buffer.readUnsignedShort();
} }
return character; return character;
} }
@ -1093,10 +1141,10 @@ public class BinaryDictInputOutput {
// readDictionaryBinary is the public entry point for them. // readDictionaryBinary is the public entry point for them.
static final int[] characterBuffer = new int[MAX_WORD_LENGTH]; static final int[] characterBuffer = new int[MAX_WORD_LENGTH];
private static CharGroupInfo readCharGroup(final ByteBuffer buffer, private static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
final int originalGroupAddress) { final int originalGroupAddress) {
int addressPointer = originalGroupAddress; int addressPointer = originalGroupAddress;
final int flags = readUnsignedByte(buffer); final int flags = buffer.readUnsignedByte();
++addressPointer; ++addressPointer;
final int characters[]; final int characters[];
if (0 != (flags & FLAG_HAS_MULTIPLE_CHARS)) { if (0 != (flags & FLAG_HAS_MULTIPLE_CHARS)) {
@ -1117,22 +1165,22 @@ public class BinaryDictInputOutput {
final int frequency; final int frequency;
if (0 != (FLAG_IS_TERMINAL & flags)) { if (0 != (FLAG_IS_TERMINAL & flags)) {
++addressPointer; ++addressPointer;
frequency = readUnsignedByte(buffer); frequency = buffer.readUnsignedByte();
} else { } else {
frequency = CharGroup.NOT_A_TERMINAL; frequency = CharGroup.NOT_A_TERMINAL;
} }
int childrenAddress = addressPointer; int childrenAddress = addressPointer;
switch (flags & MASK_GROUP_ADDRESS_TYPE) { switch (flags & MASK_GROUP_ADDRESS_TYPE) {
case FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: case FLAG_GROUP_ADDRESS_TYPE_ONEBYTE:
childrenAddress += readUnsignedByte(buffer); childrenAddress += buffer.readUnsignedByte();
addressPointer += 1; addressPointer += 1;
break; break;
case FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: case FLAG_GROUP_ADDRESS_TYPE_TWOBYTES:
childrenAddress += readUnsignedShort(buffer); childrenAddress += buffer.readUnsignedShort();
addressPointer += 2; addressPointer += 2;
break; break;
case FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: case FLAG_GROUP_ADDRESS_TYPE_THREEBYTES:
childrenAddress += readUnsignedInt24(buffer); childrenAddress += buffer.readUnsignedInt24();
addressPointer += 3; addressPointer += 3;
break; break;
case FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: case FLAG_GROUP_ADDRESS_TYPE_NOADDRESS:
@ -1144,9 +1192,9 @@ public class BinaryDictInputOutput {
if (0 != (flags & FLAG_HAS_SHORTCUT_TARGETS)) { if (0 != (flags & FLAG_HAS_SHORTCUT_TARGETS)) {
final int pointerBefore = buffer.position(); final int pointerBefore = buffer.position();
shortcutTargets = new ArrayList<WeightedString>(); shortcutTargets = new ArrayList<WeightedString>();
buffer.getShort(); // Skip the size buffer.readUnsignedShort(); // Skip the size
while (true) { while (true) {
final int targetFlags = readUnsignedByte(buffer); final int targetFlags = buffer.readUnsignedByte();
final String word = CharEncoding.readString(buffer); final String word = CharEncoding.readString(buffer);
shortcutTargets.add(new WeightedString(word, shortcutTargets.add(new WeightedString(word,
targetFlags & FLAG_ATTRIBUTE_FREQUENCY)); targetFlags & FLAG_ATTRIBUTE_FREQUENCY));
@ -1158,22 +1206,22 @@ public class BinaryDictInputOutput {
if (0 != (flags & FLAG_HAS_BIGRAMS)) { if (0 != (flags & FLAG_HAS_BIGRAMS)) {
bigrams = new ArrayList<PendingAttribute>(); bigrams = new ArrayList<PendingAttribute>();
while (true) { while (true) {
final int bigramFlags = readUnsignedByte(buffer); final int bigramFlags = buffer.readUnsignedByte();
++addressPointer; ++addressPointer;
final int sign = 0 == (bigramFlags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) ? 1 : -1; final int sign = 0 == (bigramFlags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) ? 1 : -1;
int bigramAddress = addressPointer; int bigramAddress = addressPointer;
switch (bigramFlags & MASK_ATTRIBUTE_ADDRESS_TYPE) { switch (bigramFlags & MASK_ATTRIBUTE_ADDRESS_TYPE) {
case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE:
bigramAddress += sign * readUnsignedByte(buffer); bigramAddress += sign * buffer.readUnsignedByte();
addressPointer += 1; addressPointer += 1;
break; break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES:
bigramAddress += sign * readUnsignedShort(buffer); bigramAddress += sign * buffer.readUnsignedShort();
addressPointer += 2; addressPointer += 2;
break; break;
case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES:
final int offset = (readUnsignedByte(buffer) << 16) final int offset = (buffer.readUnsignedByte() << 16)
+ readUnsignedShort(buffer); + buffer.readUnsignedShort();
bigramAddress += sign * offset; bigramAddress += sign * offset;
addressPointer += 3; addressPointer += 3;
break; break;
@ -1192,13 +1240,13 @@ public class BinaryDictInputOutput {
/** /**
* Reads and returns the char group count out of a buffer and forwards the pointer. * Reads and returns the char group count out of a buffer and forwards the pointer.
*/ */
private static int readCharGroupCount(final ByteBuffer buffer) { private static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
final int msb = readUnsignedByte(buffer); final int msb = buffer.readUnsignedByte();
if (MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { if (MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
return msb; return msb;
} else { } else {
return ((MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) return ((MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8)
+ readUnsignedByte(buffer); + buffer.readUnsignedByte();
} }
} }
@ -1215,8 +1263,8 @@ public class BinaryDictInputOutput {
* @param address the address to seek. * @param address the address to seek.
* @return the word, as a string. * @return the word, as a string.
*/ */
private static String getWordAtAddress(final ByteBuffer buffer, final int headerSize, private static String getWordAtAddress(final FusionDictionaryBufferInterface buffer,
final int address) { final int headerSize, final int address) {
final String cachedString = wordCache.get(address); final String cachedString = wordCache.get(address);
if (null != cachedString) return cachedString; if (null != cachedString) return cachedString;
final int originalPointer = buffer.position(); final int originalPointer = buffer.position();
@ -1241,7 +1289,7 @@ public class BinaryDictInputOutput {
builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize); buffer.position(last.mChildrenAddress + headerSize);
groupOffset = last.mChildrenAddress + 1; groupOffset = last.mChildrenAddress + 1;
i = readUnsignedByte(buffer); i = buffer.readUnsignedByte();
last = null; last = null;
continue; continue;
} }
@ -1251,7 +1299,7 @@ public class BinaryDictInputOutput {
builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); builder.append(new String(last.mCharacters, 0, last.mCharacters.length));
buffer.position(last.mChildrenAddress + headerSize); buffer.position(last.mChildrenAddress + headerSize);
groupOffset = last.mChildrenAddress + 1; groupOffset = last.mChildrenAddress + 1;
i = readUnsignedByte(buffer); i = buffer.readUnsignedByte();
last = null; last = null;
continue; continue;
} }
@ -1262,10 +1310,10 @@ public class BinaryDictInputOutput {
} }
/** /**
* Reads a single node from a binary file. * Reads a single node from a buffer.
* *
* This methods reads the file at the current position of its file pointer. A node is * This methods reads the file at the current position. A node is fully expected to start at
* fully expected to start at the current position. * the current position.
* This will recursively read other nodes into the structure, populating the reverse * This will recursively read other nodes into the structure, populating the reverse
* maps on the fly and using them to keep track of already read nodes. * maps on the fly and using them to keep track of already read nodes.
* *
@ -1275,7 +1323,7 @@ public class BinaryDictInputOutput {
* @param reverseGroupMap a mapping from addresses to already read character groups. * @param reverseGroupMap a mapping from addresses to already read character groups.
* @return the read node with all his children already read. * @return the read node with all his children already read.
*/ */
private static Node readNode(final ByteBuffer buffer, final int headerSize, private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize,
final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap) final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap)
throws IOException { throws IOException {
final int nodeOrigin = buffer.position() - headerSize; final int nodeOrigin = buffer.position() - headerSize;
@ -1323,42 +1371,49 @@ public class BinaryDictInputOutput {
* Helper function to get the binary format version from the header. * Helper function to get the binary format version from the header.
* @throws IOException * @throws IOException
*/ */
private static int getFormatVersion(final ByteBuffer buffer) throws IOException { private static int getFormatVersion(final FusionDictionaryBufferInterface buffer)
final int magic_v1 = readUnsignedShort(buffer); throws IOException {
if (VERSION_1_MAGIC_NUMBER == magic_v1) return readUnsignedByte(buffer); final int magic_v1 = buffer.readUnsignedShort();
final int magic_v2 = (magic_v1 << 16) + readUnsignedShort(buffer); if (VERSION_1_MAGIC_NUMBER == magic_v1) return buffer.readUnsignedByte();
if (VERSION_2_MAGIC_NUMBER == magic_v2) return readUnsignedShort(buffer); final int magic_v2 = (magic_v1 << 16) + buffer.readUnsignedShort();
if (VERSION_2_MAGIC_NUMBER == magic_v2) return buffer.readUnsignedShort();
return NOT_A_VERSION_NUMBER; return NOT_A_VERSION_NUMBER;
} }
/** /**
* Reads options from a file and populate a map with their contents. * Reads options from a buffer and populate a map with their contents.
* *
* The file is read at the current file pointer, so the caller must take care the pointer * The buffer is read at the current position, so the caller must take care the pointer
* is in the right place before calling this. * is in the right place before calling this.
*/ */
public static void populateOptions(final ByteBuffer buffer, final int headerSize, public static void populateOptions(final FusionDictionaryBufferInterface buffer,
final HashMap<String, String> options) { final int headerSize, final HashMap<String, String> options) {
while (buffer.position() < headerSize) { while (buffer.position() < headerSize) {
final String key = CharEncoding.readString(buffer); final String key = CharEncoding.readString(buffer);
final String value = CharEncoding.readString(buffer); final String value = CharEncoding.readString(buffer);
options.put(key, value); options.put(key, value);
} }
} }
// TODO: remove this method.
public static void populateOptions(final ByteBuffer buffer, final int headerSize,
final HashMap<String, String> options) {
populateOptions(new ByteBufferWrapper(buffer), headerSize, options);
}
/** /**
* Reads a byte buffer and returns the memory representation of the dictionary. * Reads a buffer and returns the memory representation of the dictionary.
* *
* This high-level method takes a binary file and reads its contents, populating a * This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure. The optional dict argument is an existing dictionary to * FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the file should be added. If it is null, a new dictionary is created. * which words from the buffer should be added. If it is null, a new dictionary is created.
* *
* @param buffer the buffer to read. * @param buffer the buffer to read.
* @param dict an optional dictionary to add words to, or null. * @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary. * @return the created (or merged) dictionary.
*/ */
public static FusionDictionary readDictionaryBinary(final ByteBuffer buffer, public static FusionDictionary readDictionaryBinary(
final FusionDictionary dict) throws IOException, UnsupportedFormatException { final FusionDictionaryBufferInterface buffer, final FusionDictionary dict)
throws IOException, UnsupportedFormatException {
// Check file version // Check file version
final int version = getFormatVersion(buffer); final int version = getFormatVersion(buffer);
if (version < MINIMUM_SUPPORTED_VERSION || version > MAXIMUM_SUPPORTED_VERSION) { if (version < MINIMUM_SUPPORTED_VERSION || version > MAXIMUM_SUPPORTED_VERSION) {
@ -1371,14 +1426,14 @@ public class BinaryDictInputOutput {
wordCache.clear(); wordCache.clear();
// Read options // Read options
final int optionsFlags = readUnsignedShort(buffer); final int optionsFlags = buffer.readUnsignedShort();
final int headerSize; final int headerSize;
final HashMap<String, String> options = new HashMap<String, String>(); final HashMap<String, String> options = new HashMap<String, String>();
if (version < FIRST_VERSION_WITH_HEADER_SIZE) { if (version < FIRST_VERSION_WITH_HEADER_SIZE) {
headerSize = buffer.position(); headerSize = buffer.position();
} else { } else {
headerSize = buffer.getInt(); headerSize = buffer.readInt();
populateOptions(buffer, headerSize, options); populateOptions(buffer, headerSize, options);
buffer.position(headerSize); buffer.position(headerSize);
} }
@ -1413,26 +1468,10 @@ public class BinaryDictInputOutput {
return newDict; return newDict;
} }
/** // TODO: remove this method.
* Helper function to read one byte from ByteBuffer. public static FusionDictionary readDictionaryBinary(final ByteBuffer buffer,
*/ final FusionDictionary dict) throws IOException, UnsupportedFormatException {
private static int readUnsignedByte(final ByteBuffer buffer) { return readDictionaryBinary(new ByteBufferWrapper(buffer), dict);
return ((int)buffer.get()) & 0xFF;
}
/**
* Helper function to read two byte from ByteBuffer.
*/
private static int readUnsignedShort(final ByteBuffer buffer) {
return ((int)buffer.getShort()) & 0xFFFF;
}
/**
* Helper function to read three byte from ByteBuffer.
*/
private static int readUnsignedInt24(final ByteBuffer buffer) {
final int value = readUnsignedByte(buffer) << 16;
return value + readUnsignedShort(buffer);
} }
/** /**
@ -1450,7 +1489,7 @@ public class BinaryDictInputOutput {
inStream = new FileInputStream(file); inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map( final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length()); FileChannel.MapMode.READ_ONLY, 0, file.length());
final int version = getFormatVersion(buffer); final int version = getFormatVersion(new ByteBufferWrapper(buffer));
return (version >= MINIMUM_SUPPORTED_VERSION && version <= MAXIMUM_SUPPORTED_VERSION); return (version >= MINIMUM_SUPPORTED_VERSION && version <= MAXIMUM_SUPPORTED_VERSION);
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
return false; return false;