nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am 11cccb56: am d188af70: Add SparseTable. 

* commit '11cccb569f2d40caee07e7f1c8c9f95bfdeb9d42':
  Add SparseTable.
main
Yuichiro Hanada 2013-10-03 10:20:27 -07:00 committed by Android Git Automerger
parent 9cb1b71c30 11cccb569f
commit 09129f58f2
3 changed files with 330 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
View File

@ -225,6 +225,26 @@ public class BinaryDictEncoderUtils {
return position; return position;
} }
static void writeUIntToStream(final OutputStream stream, final int value, final int size)
throws IOException {
switch(size) {
case 4:
stream.write((value >> 24) & 0xFF);
/* fall through */
case 3:
stream.write((value >> 16) & 0xFF);
/* fall through */
case 2:
stream.write((value >> 8) & 0xFF);
/* fall through */
case 1:
stream.write(value & 0xFF);
break;
default:
/* nop */
}
}
// End utility methods // End utility methods
// This method is responsible for finding a nice ordering of the nodes that favors run-time // This method is responsible for finding a nice ordering of the nodes that favors run-time

150
java/src/com/android/inputmethod/latin/makedict/SparseTable.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
/**
* SparseTable is an extensible map from integer to integer.
* This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index
* memory.
*/
@UsedForTesting
public class SparseTable {
/**
* mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize
* terminals.
* It contains at index i = j / mBlockSize the index in mContentsTable where the values for
* terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array.
*/
private final ArrayList<Integer> mLookupTable;
private final ArrayList<Integer> mContentTable;
private final int mBlockSize;
public static final int NOT_EXIST = -1;
@UsedForTesting
public SparseTable(final int initialCapacity, final int blockSize) {
mBlockSize = blockSize;
final int lookupTableSize = initialCapacity / mBlockSize
+ (initialCapacity % mBlockSize > 0 ? 1 : 0);
mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST));
mContentTable = new ArrayList<Integer>();
}
@UsedForTesting
public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) {
mBlockSize = blockSize;
mLookupTable = new ArrayList<Integer>(lookupTable.length);
for (int i = 0; i < lookupTable.length; ++i) {
mLookupTable.add(lookupTable[i]);
}
mContentTable = new ArrayList<Integer>(contentTable.length);
for (int i = 0; i < contentTable.length; ++i) {
mContentTable.add(contentTable[i]);
}
}
/**
* Converts an byte array to an int array considering each set of 4 bytes is an int stored in
* big-endian.
* The length of byteArray must be a multiple of four.
* Otherwise, IndexOutOfBoundsException will be raised.
*/
@UsedForTesting
private static void convertByteArrayToIntegerArray(final byte[] byteArray,
final ArrayList<Integer> integerArray) {
for (int i = 0; i < byteArray.length; i += 4) {
int value = 0;
for (int j = i; j < i + 4; ++j) {
value <<= 8;
value |= byteArray[j] & 0xFF;
}
integerArray.add(value);
}
}
@UsedForTesting
public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) {
mBlockSize = blockSize;
mLookupTable = new ArrayList<Integer>(lookupTable.length / 4);
mContentTable = new ArrayList<Integer>(contentTable.length / 4);
convertByteArrayToIntegerArray(lookupTable, mLookupTable);
convertByteArrayToIntegerArray(contentTable, mContentTable);
}
@UsedForTesting
public int get(final int index) {
if (index < 0 || index / mBlockSize >= mLookupTable.size()
|| mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
return NOT_EXIST;
}
return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
}
@UsedForTesting
public void set(final int index, final int value) {
if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
mLookupTable.set(index / mBlockSize, mContentTable.size());
for (int i = 0; i < mBlockSize; ++i) {
mContentTable.add(NOT_EXIST);
}
}
mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
}
public void remove(final int index) {
set(index, NOT_EXIST);
}
@UsedForTesting
public int size() {
return mLookupTable.size() * mBlockSize;
}
@UsedForTesting
/* package */ int getContentTableSize() {
return mContentTable.size();
}
@UsedForTesting
/* package */ int getLookupTableSize() {
return mLookupTable.size();
}
public boolean contains(final int index) {
return get(index) != NOT_EXIST;
}
@UsedForTesting
public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream)
throws IOException {
for (final int index : mLookupTable) {
BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4);
}
for (final int index : mContentTable) {
BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
}
}
}

160
tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java Normal file
View File

@ -0,0 +1,160 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Random;
/**
* Unit tests for SparseTable.
*/
@LargeTest
public class SparseTableTests extends AndroidTestCase {
private static final String TAG = SparseTableTests.class.getSimpleName();
private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 };
private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7};
private final Random mRandom;
private final ArrayList<Integer> mRandomIndex;
private static final int DEFAULT_SIZE = 10000;
private static final int BLOCK_SIZE = 8;
public SparseTableTests() {
this(System.currentTimeMillis(), DEFAULT_SIZE);
}
public SparseTableTests(final long seed, final int tableSize) {
super();
Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize);
mRandom = new Random(seed);
mRandomIndex = new ArrayList<Integer>(tableSize);
for (int i = 0; i < tableSize; ++i) {
mRandomIndex.add(SparseTable.NOT_EXIST);
}
}
public void testInitializeWithArray() {
final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE);
for (int i = 0; i < 8; ++i) {
assertEquals(SparseTable.NOT_EXIST, table.get(i));
}
assertEquals(SparseTable.NOT_EXIST, table.get(8));
for (int i = 9; i < 16; ++i) {
assertEquals(i - 8, table.get(i));
}
}
public void testSet() {
final SparseTable table = new SparseTable(16, BLOCK_SIZE);
table.set(3, 6);
table.set(8, 16);
for (int i = 0; i < 16; ++i) {
if (i == 3 || i == 8) {
assertEquals(i * 2, table.get(i));
} else {
assertEquals(SparseTable.NOT_EXIST, table.get(i));
}
}
}
private void generateRandomIndex(final int size, final int prop) {
for (int i = 0; i < DEFAULT_SIZE; ++i) {
if (mRandom.nextInt(100) < prop) {
mRandomIndex.set(i, mRandom.nextInt());
} else {
mRandomIndex.set(i, SparseTable.NOT_EXIST);
}
}
}
private void runTestRandomSet() {
final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE);
int elementCount = 0;
for (int i = 0; i < DEFAULT_SIZE; ++i) {
if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
table.set(i, mRandomIndex.get(i));
elementCount++;
}
}
Log.d(TAG, "table size = " + table.getLookupTableSize() + " + "
+ table.getContentTableSize());
Log.d(TAG, "the table has " + elementCount + " elements");
for (int i = 0; i < DEFAULT_SIZE; ++i) {
assertEquals(table.get(i), (int)mRandomIndex.get(i));
}
// flush and reload
OutputStream lookupOutStream = null;
OutputStream contentOutStream = null;
InputStream lookupInStream = null;
InputStream contentInStream = null;
try {
final File lookupIndexFile = File.createTempFile("testRandomSet", ".small");
final File contentFile = File.createTempFile("testRandomSet", ".big");
lookupOutStream = new FileOutputStream(lookupIndexFile);
contentOutStream = new FileOutputStream(contentFile);
table.write(lookupOutStream, contentOutStream);
lookupInStream = new FileInputStream(lookupIndexFile);
contentInStream = new FileInputStream(contentFile);
final byte[] lookupArray = new byte[(int) lookupIndexFile.length()];
final byte[] contentArray = new byte[(int) contentFile.length()];
lookupInStream.read(lookupArray);
contentInStream.read(contentArray);
final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE);
for (int i = 0; i < DEFAULT_SIZE; ++i) {
assertEquals(table.get(i), newTable.get(i));
}
} catch (IOException e) {
Log.d(TAG, "IOException while flushing and realoding", e);
} finally {
if (lookupOutStream != null) {
try {
lookupOutStream.close();
} catch (IOException e) {
Log.d(TAG, "IOException while closing the stream", e);
}
}
if (contentOutStream != null) {
try {
contentOutStream.close();
} catch (IOException e) {
Log.d(TAG, "IOException while closing contentStream.", e);
}
}
}
}
public void testRandomSet() {
for (int i = 0; i <= 100; i += 10) {
generateRandomIndex(DEFAULT_SIZE, i);
runTestRandomSet();
}
}
}