/* * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.decoder.HeaderReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.TreeMap; /** * Decodes binary files for a FusionDictionary. * * All the methods in this class are static. * * TODO: Remove calls from classes except BinaryDictDecoder * TODO: Move this file to makedict/internal. */ public final class BinaryDictDecoderUtils { private static final boolean DBG = MakedictLog.DBG; private BinaryDictDecoderUtils() { // This utility class is not publicly instantiable. } private static final int MAX_JUMPS = 12; @UsedForTesting public interface DictBuffer { public int readUnsignedByte(); public int readUnsignedShort(); public int readUnsignedInt24(); public int readInt(); public int position(); public void position(int newPosition); public void put(final byte b); public int limit(); @UsedForTesting public int capacity(); } public static final class ByteBufferDictBuffer implements DictBuffer { private ByteBuffer mBuffer; public ByteBufferDictBuffer(final ByteBuffer buffer) { mBuffer = buffer; } @Override public int readUnsignedByte() { return mBuffer.get() & 0xFF; } @Override public int readUnsignedShort() { return mBuffer.getShort() & 0xFFFF; } @Override public int readUnsignedInt24() { final int retval = readUnsignedByte(); return (retval << 16) + readUnsignedShort(); } @Override public int readInt() { return mBuffer.getInt(); } @Override public int position() { return mBuffer.position(); } @Override public void position(int newPos) { mBuffer.position(newPos); } @Override public void put(final byte b) { mBuffer.put(b); } @Override public int limit() { return mBuffer.limit(); } @Override public int capacity() { return mBuffer.capacity(); } } /** * A class grouping utility function for our specific character encoding. */ static final class CharEncoding { private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; /** * Helper method to find out whether this code fits on one byte */ private static boolean fitsOnOneByte(final int character) { return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; } /** * Compute the size of a character given its character code. * * Char format is: * 1 byte = bbbbbbbb match * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with * 00011111 would be outside unicode. * else: iso-latin-1 code * This allows for the whole unicode range to be encoded, including chars outside of * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control * characters which should never happen anyway (and still work, but take 3 bytes). * * @param character the character code. * @return the size in binary encoded-form, either 1 or 3 bytes. */ static int getCharSize(final int character) { // See char encoding in FusionDictionary.java if (fitsOnOneByte(character)) return 1; if (FormatSpec.INVALID_CHARACTER == character) return 1; return 3; } /** * Compute the byte size of a character array. */ static int getCharArraySize(final int[] chars) { int size = 0; for (int character : chars) size += getCharSize(character); return size; } /** * Writes a char array to a byte buffer. * * @param codePoints the code point array to write. * @param buffer the byte buffer to write to. * @param index the index in buffer to write the character array to. * @return the index after the last character. */ static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { for (int codePoint : codePoints) { if (1 == getCharSize(codePoint)) { buffer[index++] = (byte)codePoint; } else { buffer[index++] = (byte)(0xFF & (codePoint >> 16)); buffer[index++] = (byte)(0xFF & (codePoint >> 8)); buffer[index++] = (byte)(0xFF & codePoint); } } return index; } /** * Writes a string with our character format to a byte buffer. * * This will also write the terminator byte. * * @param buffer the byte buffer to write to. * @param origin the offset to write from. * @param word the string to write. * @return the size written, in bytes. */ static int writeString(final byte[] buffer, final int origin, final String word) { final int length = word.length(); int index = origin; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); if (1 == getCharSize(codePoint)) { buffer[index++] = (byte)codePoint; } else { buffer[index++] = (byte)(0xFF & (codePoint >> 16)); buffer[index++] = (byte)(0xFF & (codePoint >> 8)); buffer[index++] = (byte)(0xFF & codePoint); } } buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; return index - origin; } /** * Writes a string with our character format to a ByteArrayOutputStream. * * This will also write the terminator byte. * * @param buffer the ByteArrayOutputStream to write to. * @param word the string to write. */ static void writeString(final ByteArrayOutputStream buffer, final String word) { final int length = word.length(); for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); if (1 == getCharSize(codePoint)) { buffer.write((byte) codePoint); } else { buffer.write((byte) (0xFF & (codePoint >> 16))); buffer.write((byte) (0xFF & (codePoint >> 8))); buffer.write((byte) (0xFF & codePoint)); } } buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); } /** * Reads a string from a DictBuffer. This is the converse of the above method. */ static String readString(final DictBuffer dictBuffer) { final StringBuilder s = new StringBuilder(); int character = readChar(dictBuffer); while (character != FormatSpec.INVALID_CHARACTER) { s.appendCodePoint(character); character = readChar(dictBuffer); } return s.toString(); } /** * Reads a character from the buffer. * * This follows the character format documented earlier in this source file. * * @param dictBuffer the buffer, positioned over an encoded character. * @return the character code. */ static int readChar(final DictBuffer dictBuffer) { int character = dictBuffer.readUnsignedByte(); if (!fitsOnOneByte(character)) { if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { return FormatSpec.INVALID_CHARACTER; } character <<= 16; character += dictBuffer.readUnsignedShort(); } return character; } } // Input methods: Read a binary dictionary to memory. // readDictionaryBinary is the public entry point for them. static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { if (options.mSupportsDynamicUpdate) { final int address = dictBuffer.readUnsignedInt24(); if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; if ((address & FormatSpec.MSB24) != 0) { return -(address & FormatSpec.SINT24_MAX); } else { return address; } } int address; switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: return dictBuffer.readUnsignedByte(); case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: return dictBuffer.readUnsignedShort(); case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: return dictBuffer.readUnsignedInt24(); case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: default: return FormatSpec.NO_CHILDREN_ADDRESS; } } static int readParentAddress(final DictBuffer dictBuffer, final FormatOptions formatOptions) { if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { final int parentAddress = dictBuffer.readUnsignedInt24(); final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; return sign * (parentAddress & FormatSpec.SINT24_MAX); } else { return FormatSpec.NO_PARENT_ADDRESS; } } private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; public static CharGroupInfo readCharGroup(final DictBuffer dictBuffer, final int originalGroupAddress, final FormatOptions options) { int addressPointer = originalGroupAddress; final int flags = dictBuffer.readUnsignedByte(); ++addressPointer; final int parentAddress = readParentAddress(dictBuffer, options); if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { addressPointer += 3; } final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; int character = CharEncoding.readChar(dictBuffer); addressPointer += CharEncoding.getCharSize(character); while (-1 != character) { // FusionDictionary is making sure that the length of the word is smaller than // MAX_WORD_LENGTH. // So we'll never write past the end of CHARACTER_BUFFER. CHARACTER_BUFFER[index++] = character; character = CharEncoding.readChar(dictBuffer); addressPointer += CharEncoding.getCharSize(character); } characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); } else { final int character = CharEncoding.readChar(dictBuffer); addressPointer += CharEncoding.getCharSize(character); characters = new int[] { character }; } final int frequency; if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { ++addressPointer; frequency = dictBuffer.readUnsignedByte(); } else { frequency = CharGroup.NOT_A_TERMINAL; } int childrenAddress = readChildrenAddress(dictBuffer, flags, options); if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { childrenAddress += addressPointer; } addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); ArrayList shortcutTargets = null; if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { final int pointerBefore = dictBuffer.position(); shortcutTargets = new ArrayList(); dictBuffer.readUnsignedShort(); // Skip the size while (true) { final int targetFlags = dictBuffer.readUnsignedByte(); final String word = CharEncoding.readString(dictBuffer); shortcutTargets.add(new WeightedString(word, targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; } addressPointer += dictBuffer.position() - pointerBefore; } ArrayList bigrams = null; if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { bigrams = new ArrayList(); int bigramCount = 0; while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { final int bigramFlags = dictBuffer.readUnsignedByte(); ++addressPointer; final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) ? 1 : -1; int bigramAddress = addressPointer; switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: bigramAddress += sign * dictBuffer.readUnsignedByte(); addressPointer += 1; break; case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: bigramAddress += sign * dictBuffer.readUnsignedShort(); addressPointer += 2; break; case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: final int offset = (dictBuffer.readUnsignedByte() << 16) + dictBuffer.readUnsignedShort(); bigramAddress += sign * offset; addressPointer += 3; break; default: throw new RuntimeException("Has bigrams with no address"); } bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, bigramAddress)); if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; } if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { MakedictLog.d("too many bigrams in a group."); } } return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, parentAddress, childrenAddress, shortcutTargets, bigrams); } /** * Reads and returns the char group count out of a buffer and forwards the pointer. */ public static int readCharGroupCount(final DictBuffer dictBuffer) { final int msb = dictBuffer.readUnsignedByte(); if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { return msb; } else { return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) + dictBuffer.readUnsignedByte(); } } /** * Finds, as a string, the word at the address passed as an argument. * * @param dictBuffer the buffer to read from. * @param headerSize the size of the header. * @param address the address to seek. * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ /* package for tests */ static WeightedString getWordAtAddress( final DictBuffer dictBuffer, final int headerSize, final int address, final FormatOptions formatOptions) { final WeightedString result; final int originalPointer = dictBuffer.position(); dictBuffer.position(address); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { result = getWordAtAddressWithParentAddress(dictBuffer, headerSize, address, formatOptions); } else { result = getWordAtAddressWithoutParentAddress(dictBuffer, headerSize, address, formatOptions); } dictBuffer.position(originalPointer); return result; } @SuppressWarnings("unused") private static WeightedString getWordAtAddressWithParentAddress( final DictBuffer dictBuffer, final int headerSize, final int address, final FormatOptions options) { int currentAddress = address; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { CharGroupInfo currentInfo; int loopCounter = 0; do { dictBuffer.position(currentAddress + headerSize); currentInfo = readCharGroup(dictBuffer, currentAddress, options); if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } if (DBG && loopCounter++ > MAX_JUMPS) { MakedictLog.d("Too many jumps - probably a bug"); } } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)); if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; builder.insert(0, new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } return new WeightedString(builder.toString(), frequency); } private static WeightedString getWordAtAddressWithoutParentAddress( final DictBuffer dictBuffer, final int headerSize, final int address, final FormatOptions options) { dictBuffer.position(headerSize); final int count = readCharGroupCount(dictBuffer); int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); final StringBuilder builder = new StringBuilder(); WeightedString result = null; CharGroupInfo last = null; for (int i = count - 1; i >= 0; --i) { CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); groupOffset = info.mEndAddress; if (info.mOriginalAddress == address) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); result = new WeightedString(builder.toString(), info.mFrequency); break; // and return } if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { if (info.mChildrenAddress > address) { if (null == last) continue; builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); dictBuffer.position(last.mChildrenAddress + headerSize); i = readCharGroupCount(dictBuffer); groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } last = info; } if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); dictBuffer.position(last.mChildrenAddress + headerSize); i = readCharGroupCount(dictBuffer); groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } } return result; } /** * Reads a single node array from a buffer. * * This methods reads the file at the current position. A node array is fully expected to start * at the current position. * This will recursively read other node arrays into the structure, populating the reverse * maps on the fly and using them to keep track of already read nodes. * * @param dictBuffer the buffer, correctly positioned at the start of a node array. * @param headerSize the size, in bytes, of the file header. * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. * @param reverseGroupMap a mapping from addresses to already read character groups. * @param options file format options. * @return the read node array with all his children already read. */ private static PtNodeArray readNodeArray(final DictBuffer dictBuffer, final int headerSize, final Map reverseNodeArrayMap, final Map reverseGroupMap, final FormatOptions options) throws IOException { final ArrayList nodeArrayContents = new ArrayList(); final int nodeArrayOrigin = dictBuffer.position() - headerSize; do { // Scan the linked-list node. final int nodeArrayHeadPosition = dictBuffer.position() - headerSize; final int count = readCharGroupCount(dictBuffer); int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of CharGroup. CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; ArrayList shortcutTargets = info.mShortcutTargets; ArrayList bigrams = null; if (null != info.mBigrams) { bigrams = new ArrayList(); for (PendingAttribute bigram : info.mBigrams) { final WeightedString word = getWordAtAddress( dictBuffer, headerSize, bigram.mAddress, options); final int reconstructedFrequency = BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); } } if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); if (null == children) { final int currentPosition = dictBuffer.position(); dictBuffer.position(info.mChildrenAddress + headerSize); children = readNodeArray(dictBuffer, headerSize, reverseNodeArrayMap, reverseGroupMap, options); dictBuffer.position(currentPosition); } nodeArrayContents.add( new CharGroup(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); } else { nodeArrayContents.add( new CharGroup(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); } groupOffset = info.mEndAddress; } // reach the end of the array. if (options.mSupportsDynamicUpdate) { final int nextAddress = dictBuffer.readUnsignedInt24(); if (nextAddress >= 0 && nextAddress < dictBuffer.limit()) { dictBuffer.position(nextAddress); } else { break; } } } while (options.mSupportsDynamicUpdate && dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); nodeArray.mCachedAddressBeforeUpdate = nodeArrayOrigin; nodeArray.mCachedAddressAfterUpdate = nodeArrayOrigin; reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); return nodeArray; } /** * Helper function to get the binary format version from the header. * @throws IOException */ private static int getFormatVersion(final DictBuffer dictBuffer) throws IOException { final int magic = dictBuffer.readInt(); if (FormatSpec.MAGIC_NUMBER == magic) return dictBuffer.readUnsignedShort(); return FormatSpec.NOT_A_VERSION_NUMBER; } /** * Helper function to get and validate the binary format version. * @throws UnsupportedFormatException * @throws IOException */ static int checkFormatVersion(final DictBuffer dictBuffer) throws IOException, UnsupportedFormatException { final int version = getFormatVersion(dictBuffer); if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { throw new UnsupportedFormatException("This file has version " + version + ", but this implementation does not support versions above " + FormatSpec.MAXIMUM_SUPPORTED_VERSION); } return version; } /** * Reads a header from a buffer. * @param headerReader the header reader * @throws IOException * @throws UnsupportedFormatException */ public static FileHeader readHeader(final HeaderReader headerReader) throws IOException, UnsupportedFormatException { final int version = headerReader.readVersion(); final int optionsFlags = headerReader.readOptionFlags(); final int headerSize = headerReader.readHeaderSize(); if (headerSize < 0) { throw new UnsupportedFormatException("header size can't be negative."); } final HashMap attributes = headerReader.readAttributes(headerSize); final FileHeader header = new FileHeader(headerSize, new FusionDictionary.DictionaryOptions(attributes, 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), new FormatOptions(version, 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); return header; } /** * Reads options from a buffer and populate a map with their contents. * * The buffer is read at the current position, so the caller must take care the pointer * is in the right place before calling this. */ public static void populateOptions(final DictBuffer dictBuffer, final int headerSize, final HashMap options) { while (dictBuffer.position() < headerSize) { final String key = CharEncoding.readString(dictBuffer); final String value = CharEncoding.readString(dictBuffer); options.put(key, value); } } /** * Reads a buffer and returns the memory representation of the dictionary. * * This high-level method takes a buffer and reads its contents, populating a * FusionDictionary structure. The optional dict argument is an existing dictionary to * which words from the buffer should be added. If it is null, a new dictionary is created. * * @param dictDecoder the dict decoder. * @param dict an optional dictionary to add words to, or null. * @return the created (or merged) dictionary. */ @UsedForTesting public static FusionDictionary readDictionaryBinary(final BinaryDictDecoder dictDecoder, final FusionDictionary dict) throws FileNotFoundException, IOException, UnsupportedFormatException { // if the buffer has not been opened, open the buffer with bytebuffer. if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer( new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); if (dictDecoder.getDictBuffer() == null) { MakedictLog.e("Cannot open the buffer"); } // Read header final FileHeader fileHeader = readHeader(dictDecoder); Map reverseNodeArrayMapping = new TreeMap(); Map reverseGroupMapping = new TreeMap(); final PtNodeArray root = readNodeArray(dictDecoder.getDictBuffer(), fileHeader.mHeaderSize, reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); if (null != dict) { for (final Word w : dict) { if (w.mIsBlacklistEntry) { newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); } else { newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); } } for (final Word w : dict) { // By construction a binary dictionary may not have bigrams pointing to // words that are not also registered as unigrams so we don't have to avoid // them explicitly here. for (final WeightedString bigram : w.mBigrams) { newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); } } } return newDict; } /** * Helper method to pass a file name instead of a File object to isBinaryDictionary. */ public static boolean isBinaryDictionary(final String filename) { final File file = new File(filename); return isBinaryDictionary(file); } /** * Basic test to find out whether the file is a binary dictionary or not. * * Concretely this only tests the magic number. * * @param file The file to test. * @return true if it's a binary dictionary, false otherwise */ public static boolean isBinaryDictionary(final File file) { FileInputStream inStream = null; try { inStream = new FileInputStream(file); final ByteBuffer buffer = inStream.getChannel().map( FileChannel.MapMode.READ_ONLY, 0, file.length()); final int version = getFormatVersion(new ByteBufferDictBuffer(buffer)); return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); } catch (FileNotFoundException e) { return false; } catch (IOException e) { return false; } finally { if (inStream != null) { try { inStream.close(); } catch (IOException e) { // do nothing } } } } }