/* * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import android.util.Log; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; /** * An implementation of DictDecoder for version 2 binary dictionary. */ @UsedForTesting public class Ver2DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver2DictDecoder.class.getSimpleName(); /** * A utility class for reading a PtNode. */ protected static class PtNodeReader { private static ProbabilityInfo readProbabilityInfo(final DictBuffer dictBuffer) { // Ver2 dicts don't contain historical information. return new ProbabilityInfo(dictBuffer.readUnsignedByte()); } protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { return dictBuffer.readUnsignedByte(); } protected static int readChildrenAddress(final DictBuffer dictBuffer, final int ptNodeFlags) { switch (ptNodeFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return dictBuffer.readUnsignedByte(); case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: return dictBuffer.readUnsignedShort(); case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: return dictBuffer.readUnsignedInt24(); case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: default: return FormatSpec.NO_CHILDREN_ADDRESS; } } // Reads shortcuts and returns the read length. protected static int readShortcut(final DictBuffer dictBuffer, final ArrayList shortcutTargets) { final int pointerBefore = dictBuffer.position(); dictBuffer.readUnsignedShort(); // skip the size while (true) { final int targetFlags = dictBuffer.readUnsignedByte(); final String word = CharEncoding.readString(dictBuffer); shortcutTargets.add(new WeightedString(word, targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; } return dictBuffer.position() - pointerBefore; } protected static int readBigramAddresses(final DictBuffer dictBuffer, final ArrayList bigrams, final int baseAddress) { int readLength = 0; int bigramCount = 0; while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { final int bigramFlags = dictBuffer.readUnsignedByte(); ++readLength; final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) ? 1 : -1; int bigramAddress = baseAddress + readLength; switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: bigramAddress += sign * dictBuffer.readUnsignedByte(); readLength += 1; break; case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: bigramAddress += sign * dictBuffer.readUnsignedShort(); readLength += 2; break; case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: bigramAddress += sign * dictBuffer.readUnsignedInt24(); readLength += 3; break; default: throw new RuntimeException("Has bigrams with no address"); } bigrams.add(new PendingAttribute( bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, bigramAddress)); if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; } return readLength; } } protected final File mDictionaryBinaryFile; private final DictionaryBufferFactory mBufferFactory; protected DictBuffer mDictBuffer; /* package */ Ver2DictDecoder(final File file, final int factoryFlag) { mDictionaryBinaryFile = file; mDictBuffer = null; if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { mBufferFactory = new DictionaryBufferFromByteArrayFactory(); } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); } else { mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); } } /* package */ Ver2DictDecoder(final File file, final DictionaryBufferFactory factory) { mDictionaryBinaryFile = file; mBufferFactory = factory; } @Override public void openDictBuffer() throws FileNotFoundException, IOException { mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); } @Override public boolean isDictBufferOpen() { return mDictBuffer != null; } /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } @UsedForTesting /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { openDictBuffer(); return getDictBuffer(); } @Override public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException { if (mDictBuffer == null) { openDictBuffer(); } final DictionaryHeader header = super.readHeader(mDictBuffer); final int version = header.mFormatOptions.mVersion; if (version != FormatSpec.VERSION2) { throw new UnsupportedFormatException("File header has a wrong version : " + version); } return header; } // TODO: Make this buffer multi thread safe. private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; @Override public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { int addressPointer = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; int character = CharEncoding.readChar(mDictBuffer); addressPointer += CharEncoding.getCharSize(character); while (FormatSpec.INVALID_CHARACTER != character) { // FusionDictionary is making sure that the length of the word is smaller than // MAX_WORD_LENGTH. // So we'll never write past the end of mCharacterBuffer. mCharacterBuffer[index++] = character; character = CharEncoding.readChar(mDictBuffer); addressPointer += CharEncoding.getCharSize(character); } characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); } else { final int character = CharEncoding.readChar(mDictBuffer); addressPointer += CharEncoding.getCharSize(character); characters = new int[] { character }; } final ProbabilityInfo probabilityInfo; if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { probabilityInfo = PtNodeReader.readProbabilityInfo(mDictBuffer); addressPointer += FormatSpec.PTNODE_FREQUENCY_SIZE; } else { probabilityInfo = null; } int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags); if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { childrenAddress += addressPointer; } addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); final ArrayList shortcutTargets; if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { // readShortcut will add shortcuts to shortcutTargets. shortcutTargets = new ArrayList(); addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); } else { shortcutTargets = null; } final ArrayList bigrams; if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { bigrams = new ArrayList(); addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, addressPointer); if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); } } else { bigrams = null; } return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo, FormatSpec.NO_PARENT_ADDRESS, childrenAddress, shortcutTargets, bigrams); } @Override public FusionDictionary readDictionaryBinary(final FusionDictionary dict, final boolean deleteDictIfBroken) throws FileNotFoundException, IOException, UnsupportedFormatException { if (mDictBuffer == null) { openDictBuffer(); } try { return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); } catch (IOException e) { Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { Log.e(TAG, "Failed to delete the broken dictionary."); } throw e; } catch (UnsupportedFormatException e) { Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { Log.e(TAG, "Failed to delete the broken dictionary."); } throw e; } } @Override public void setPosition(int newPos) { mDictBuffer.position(newPos); } @Override public int getPosition() { return mDictBuffer.position(); } @Override public int readPtNodeCount() { return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); } @Override public boolean readAndFollowForwardLink() { final int nextAddress = mDictBuffer.readUnsignedInt24(); if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { mDictBuffer.position(nextAddress); return true; } return false; } @Override public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } }