Fixed testing bug (#201)

* Fixed testing bug * Fixed testing bug * Fixed ut failed * Fixed ut failed * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments --------- Co-authored-by: luke.miao <[email protected]>
apache · Aug 23, 2024 · b5ec1d9 · b5ec1d9
1 parent 12394f0
commit b5ec1d9
Show file tree

Hide file tree

Showing 7 changed files with 240 additions and 44 deletions.
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
@@ -43,11 +43,19 @@
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.tsfile</groupId>
             <artifactId>tsfile</artifactId>
             <version>1.2.0-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -60,6 +68,16 @@
     </dependencies>
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <usedDependencies combine.children="append">
+                        <!-- We just need this dependency to prevent the compiler from freaking out on unused imports -->
+                        <usedDependency>ch.qos.logback:logback-classic</usedDependency>
+                    </usedDependencies>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>

diff --git a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml
@@ -16,14 +16,15 @@
     under the License.
 -->
 <configuration scan="true" scanPeriod="60 seconds">
+    <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
     <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
         <Target>System.out</Target>
         <encoder>
             <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
             <charset>utf-8</charset>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-            <level>ERROR</level>
+            <level>INFO</level>
         </filter>
     </appender>
     <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="tools">
@@ -46,4 +47,6 @@
         <appender-ref ref="stdout"/>
         <appender-ref ref="tools"/>
     </root>
+    <logger name="org.apache.tsfile.common.conf.TSFileDescriptor" level="OFF"/>
+    <logger name="org.apache.tsfile.write.TsFileWriter" level="OFF"/>
 </configuration>
diff --git a/java/tools/src/assembly/resources/tools/csv2tsfile.bat b/java/tools/src/assembly/resources/tools/csv2tsfile.bat
@@ -39,11 +39,10 @@ set CLASSPATH="%TSFILE_HOME%\lib\*"
 if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.tsfile.tools.TsFileTool
 
 set TSFILE_CONF=%TSFILE_HOME%\conf
-set "tsfile_params=-Dlogback.configurationFile=!IOTDB_CLI_CONF!\logback-cvs2tsfile.xml"
-start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > nul 2>&1"
+set "tsfile_params=-Dlogback.configurationFile=!TSFILE_CONF!\logback-cvs2tsfile.xml"
+start /B /WAIT "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*)"
 exit /b
 
-
 :err
 echo JAVA_HOME environment variable must be set!
 set ret_code=1

diff --git a/java/tools/src/assembly/tools.xml b/java/tools/src/assembly/tools.xml
@@ -25,7 +25,7 @@
         <format>dir</format>
         <format>zip</format>
     </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
+    <includeBaseDirectory>true</includeBaseDirectory>
     <dependencySets>
         <dependencySet>
             <outputDirectory>lib</outputDirectory>
@@ -45,6 +45,7 @@
         <file>
             <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.sh</source>
             <destName>tools/csv2tsfile.sh</destName>
+            <fileMode>0755</fileMode>
         </file>
         <file>
             <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.bat</source>

diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java
@@ -23,16 +23,17 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 public class SchemaParser {
 
   public static class Schema {
-    String tableName;
-    String timePrecision;
+    String tableName = "";
+    String timePrecision = "ms";
     boolean hasHeader = true;
-    String separator;
+    String separator = ",";
     String nullFormat;
-    String timeColumn;
+    String timeColumn = "";
     int timeColumnIndex = -1;
     List<IDColumns> idColumns = new ArrayList<>();
     List<Column> csvColumns = new ArrayList<>();
@@ -155,7 +156,12 @@ public static Schema parseSchema(String filePath) throws IOException {
         } else if (line.startsWith("time_precision=")) {
           schema.timePrecision = extractValue(line);
         } else if (line.startsWith("has_header=")) {
-          schema.hasHeader = Boolean.parseBoolean(extractValue(line));
+          String has_header = extractValue(line);
+          if (has_header.equals("true") || has_header.equals("false")) {
+            schema.hasHeader = Boolean.parseBoolean(has_header);
+          } else {
+            throw new IllegalArgumentException("The data format of has_header is incorrect");
+          }
         } else if (line.startsWith("separator=")) {
           schema.separator = extractValue(line);
         } else if (line.startsWith("null_format=")) {
@@ -208,7 +214,7 @@ private static void addIdColumnsIndex(Schema schema) {
     for (IDColumns idColumn : idColumnsList) {
       if (!idColumn.isDefault) {
         for (int j = 0; j < columnList.size(); j++) {
-          if (columnList.get(j).name.equals(idColumn.name)) {
+          if (Objects.equals(columnList.get(j).name, idColumn.name)) {
             idColumn.csvColumnIndex = j;
             break;
           }
@@ -244,21 +250,33 @@ private static void validateParams(SchemaParser.Schema schema) {
     if (!schema.timePrecision.equals("us")
         && !schema.timePrecision.equals("ms")
         && !schema.timePrecision.equals("ns")) {
-      throw new IllegalArgumentException("timePrecision must be us,ms or ns");
+      throw new IllegalArgumentException("The time_precision parameter only supports ms,us,ns");
     }
     if (!schema.separator.equals(",")
         && !schema.separator.equals("tab")
         && !schema.separator.equals(";")) {
       throw new IllegalArgumentException("separator must be \",\", tab, or \";\"");
     }
-    if (schema.timeColumnIndex < 0) {
-      throw new IllegalArgumentException("time_column is required");
-    }
     if (schema.tableName.isEmpty()) {
       throw new IllegalArgumentException("table_name is required");
     }
+    if (schema.idColumns.isEmpty()) {
+      throw new IllegalArgumentException("id_columns is required");
+    }
     if (schema.csvColumns.isEmpty()) {
       throw new IllegalArgumentException("csv_columns is required");
     }
+    if (schema.timeColumn.isEmpty()) {
+      throw new IllegalArgumentException("time_column is required");
+    } else if (schema.timeColumnIndex < 0) {
+      throw new IllegalArgumentException(
+          "The value " + schema.timeColumn + " of time_column is not in csv_columns");
+    }
+    for (IDColumns idColumn : schema.idColumns) {
+      if (idColumn.csvColumnIndex < 0 && !idColumn.isDefault) {
+        throw new IllegalArgumentException(
+            "The value " + idColumn.name + " of id_columns is not in csv_columns");
+      }
+    }
   }
 }