Hello,
ICU4JNI's CharsetDecoderICU doesn't handle stream decoding correctly when
there're some remaining bytes left in previous decode invocation.
The following test case fail when using ICU4JNI's charset, and pass when using
the default charset of SUN JDK (1.5.0_06).
========================================
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
import junit.framework.TestCase;
import com.ibm.icu4jni.charset.CharsetProviderICU;
public class ICUDecoderTest extends TestCase {
public void test_decode1() {
CharsetProviderICU provider = new CharsetProviderICU();
Charset cs = provider.charsetForName("UTF-8");
// uncomment following code to use RI's "UTF-8" charset
//cs = Charset.forName("UTF-8");
CharsetDecoder decoder = cs.newDecoder().onMalformedInput(
CodingErrorAction.REPLACE).onUnmappableCharacter(
CodingErrorAction.REPLACE);
// UTF8: 0xe28b93 = 0x22d3, 0xe28b94 = 0x22d4, 0xe1a093 = 0x1813
byte[] byte1 = { (byte) 0xe1, (byte) 0xa0 };
byte[] byte2 = { (byte) 0x93, (byte) 0xe2, (byte) 0x8b, (byte) 0x93,
(byte) 0xe2, (byte) 0x8b, (byte) 0x94, (byte) 0xe2 };
// 0xe1, 0xa0 should be remained
ByteBuffer buf1 = ByteBuffer.wrap(byte1);
CharBuffer out1 = CharBuffer.allocate(5);
CoderResult result1 = decoder.decode(buf1, out1, false);
out1.flip();
showCharBuffer(out1);
showRemaining(buf1);
assertEquals(2, buf1.remaining());
// 0xe2 should be remained, output should be 0x1813, 0x22d3, 0x22d4
ByteBuffer buf2 = ByteBuffer.allocate(10);
// get the remaining of buf1
buf2.put(buf1);
buf2.put(byte2);
buf2.flip();
CharBuffer out2 = CharBuffer.allocate(5);
CoderResult result2 = decoder.decode(buf2, out2, false);
out2.flip();
showCharBuffer(out2);
showRemaining(buf2);
assertEquals(1, buf2.remaining());
assertEquals(0x1813, out2.get());
assertEquals(0x22d3, out2.get());
assertEquals(0x22d4, out2.get());
}
private void showRemaining(ByteBuffer buf) {
int pos = buf.position();
if (buf.hasRemaining()) {
byte[] remains = new byte[buf.remaining()];
buf.get(remains);
for (int i = 0; i < remains.length; i++) {
System.out.println("Remaining: "
+ Integer.toHexString(remains[i]));
}
} else {
System.out.println("No remaining");
}
buf.position(pos);
}
private void showCharBuffer(CharBuffer buf) {
int pos = buf.position();
for (int i = 0; buf.position() != buf.limit(); i++) {
System.out.println("CharBuffer + i + ? = "
+ Integer.toHexString(buf.get()));
}
buf.position(pos);
}
}
