Fixed the bug that ReadWriteIOUtils does not guarantee to use UTF-8 i…

…n deserialization
apache · Jan 9, 2025 · d36093d · d36093d
1 parent 53cba29
commit d36093d
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 8 deletions.
diff --git a/NOTICE b/NOTICE
@@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/).
 
 ============================================================================
 
-TsFile project uses 4 Chinese Patents:
+TsFile project uses 2 Chinese Patents:
 * 201711384490X
 * 201711319331.1
 

diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -617,7 +617,7 @@ public static String readString(InputStream inputStream) throws IOException {
     if (readLen != strLength) {
       throw new IOException(String.format(RETURN_ERROR, strLength, readLen));
     }
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** String length's type is varInt */
@@ -633,7 +633,7 @@ public static String readVarIntString(InputStream inputStream) throws IOExceptio
     if (readLen != strLength) {
       throw new IOException(String.format(RETURN_ERROR, strLength, readLen));
     }
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** Read string from byteBuffer. */
@@ -646,7 +646,7 @@ public static String readString(ByteBuffer buffer) {
     }
     byte[] bytes = new byte[strLength];
     buffer.get(bytes, 0, strLength);
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** String length's type is varInt */
@@ -659,7 +659,7 @@ public static String readVarIntString(ByteBuffer buffer) {
     }
     byte[] bytes = new byte[strLength];
     buffer.get(bytes, 0, strLength);
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** Read string from byteBuffer with user define length. */
@@ -671,7 +671,7 @@ public static String readStringWithLength(ByteBuffer buffer, int length) {
     }
     byte[] bytes = new byte[length];
     buffer.get(bytes, 0, length);
-    return new String(bytes, 0, length);
+    return new String(bytes, 0, length, TSFileConfig.STRING_CHARSET);
   }
 
   public static ByteBuffer getByteBuffer(String s) {

diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
@@ -54,7 +54,21 @@ public void stringSerdeTest() {
     Assert.assertNotNull(result);
     Assert.assertEquals(str, result);
 
-    // 2. null value
+    // 2. Chinese value
+    str = "中文";
+    byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
+    stream = new DataOutputStream(byteArrayOutputStream);
+    try {
+      ReadWriteIOUtils.write(str, stream);
+    } catch (IOException e) {
+      fail(e.toString());
+    }
+
+    result = ReadWriteIOUtils.readString(ByteBuffer.wrap(byteArrayOutputStream.toByteArray()));
+    Assert.assertNotNull(result);
+    Assert.assertEquals(str, result);
+
+    // 3. null value
     str = null;
     byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
     stream = new DataOutputStream(byteArrayOutputStream);
@@ -153,7 +167,24 @@ public void mapSerdeTest() {
     Assert.assertNotNull(result);
     Assert.assertTrue(result.isEmpty());
 
-    // 6. null
+    // 6. key: chinese; value: chinese
+    key = "中文";
+    value = "中文";
+    map = new HashMap<>();
+    map.put(key, value);
+    byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
+    stream = new DataOutputStream(byteArrayOutputStream);
+    try {
+      ReadWriteIOUtils.write(map, stream);
+    } catch (IOException e) {
+      fail(e.toString());
+    }
+
+    result = ReadWriteIOUtils.readMap(ByteBuffer.wrap(byteArrayOutputStream.toByteArray()));
+    Assert.assertNotNull(result);
+    Assert.assertEquals(map, result);
+
+    // 7. null
     map = null;
     byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
     stream = new DataOutputStream(byteArrayOutputStream);