From f1466e6d131eefb8653ba5efd6e05ba842761050 Mon Sep 17 00:00:00 2001 From: shuwenwei <55970239+shuwenwei@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:06:56 +0800 Subject: [PATCH] Convert column name and table name to lower case (#322) * Only performs the lower case conversion in table model * modify DeviceTableModelWriter * fix ut * modify query * modify query * remove null check --- .../v4/WriteTabletWithITsFileWriter.java | 2 +- .../tsfile/file/metadata/TableSchema.java | 60 ++++++++++++---- .../read/v4/DeviceTableModelReader.java | 13 ++-- .../org/apache/tsfile/write/TsFileWriter.java | 15 ++-- .../chunk/AlignedChunkGroupWriterImpl.java | 46 +++++++++--- .../chunk/TableChunkGroupWriterImpl.java | 36 ++++++++++ .../apache/tsfile/write/record/Tablet.java | 35 +++++---- .../v4/AbstractTableModelTsFileWriter.java | 9 ++- .../write/v4/DeviceTableModelWriter.java | 2 +- .../tsfile/read/query/ResultSetTest.java | 20 +++--- .../tsfile/tableview/TableSchemaTest.java | 1 + .../tsfile/write/TsFileWriteApiTest.java | 71 +++++++++++++++++++ 12 files changed, 250 insertions(+), 60 deletions(-) create mode 100644 java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java diff --git a/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java b/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java index d7dca8310..5da781bdd 100644 --- a/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java +++ b/java/examples/src/main/java/org/apache/tsfile/v4/WriteTabletWithITsFileWriter.java @@ -106,7 +106,7 @@ public static void main(String[] args) throws IOException { tablet.addValue(rowIndex, 0, "id1_field_2"); // id2 column - tablet.addValue(rowIndex, 1, "id1_field_2"); + tablet.addValue(rowIndex, 1, "id2_field_2"); // s1 column tablet.addValue(rowIndex, 2, 1); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java index 21d444250..fcd3b4345 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TableSchema.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class TableSchema { @@ -53,18 +54,45 @@ public class TableSchema { private Map idColumnOrder; public TableSchema(String tableName) { - this.tableName = tableName; + this.tableName = tableName.toLowerCase(); this.measurementSchemas = new ArrayList<>(); this.columnCategories = new ArrayList<>(); this.updatable = true; } + // for deserialize + public TableSchema( + List columnSchemas, List columnCategories) { + this.measurementSchemas = + columnSchemas.stream() + .map( + measurementSchema -> + new MeasurementSchema( + measurementSchema.getMeasurementName().toLowerCase(), + measurementSchema.getType(), + measurementSchema.getEncodingType(), + measurementSchema.getCompressor(), + measurementSchema.getProps())) + .collect(Collectors.toList()); + this.columnCategories = columnCategories; + } + public TableSchema( String tableName, List columnSchemas, List columnCategories) { - this.tableName = tableName; - this.measurementSchemas = columnSchemas; + this.tableName = tableName.toLowerCase(); + this.measurementSchemas = + columnSchemas.stream() + .map( + measurementSchema -> + new MeasurementSchema( + measurementSchema.getMeasurementName().toLowerCase(), + measurementSchema.getType(), + measurementSchema.getEncodingType(), + measurementSchema.getCompressor(), + measurementSchema.getProps())) + .collect(Collectors.toList()); this.columnCategories = columnCategories; } @@ -73,22 +101,24 @@ public TableSchema( List columnNameList, List dataTypeList, List categoryList) { - this.tableName = tableName; + this.tableName = tableName.toLowerCase(); this.measurementSchemas = new ArrayList<>(columnNameList.size()); for (int i = 0; i < columnNameList.size(); i++) { - measurementSchemas.add(new MeasurementSchema(columnNameList.get(i), dataTypeList.get(i))); + measurementSchemas.add( + new MeasurementSchema(columnNameList.get(i).toLowerCase(), dataTypeList.get(i))); } this.columnCategories = categoryList; } @TsFileApi public TableSchema(String tableName, List columnSchemaList) { - this.tableName = tableName; + this.tableName = tableName.toLowerCase(); this.measurementSchemas = new ArrayList<>(columnSchemaList.size()); this.columnCategories = new ArrayList<>(columnSchemaList.size()); for (ColumnSchema columnSchema : columnSchemaList) { this.measurementSchemas.add( - new MeasurementSchema(columnSchema.getColumnName(), columnSchema.getDataType())); + new MeasurementSchema( + columnSchema.getColumnName().toLowerCase(), columnSchema.getDataType())); this.columnCategories.add(columnSchema.getColumnCategory()); } } @@ -126,12 +156,13 @@ public Map getIdColumnOrder() { * @return i if the given column is the i-th column, -1 if the column is not in the schema */ public int findColumnIndex(String columnName) { + final String lowerCaseColumnName = columnName.toLowerCase(); return getColumnPosIndex() .computeIfAbsent( - columnName, + lowerCaseColumnName, colName -> { for (int i = 0; i < measurementSchemas.size(); i++) { - if (measurementSchemas.get(i).getMeasurementName().equals(columnName)) { + if (measurementSchemas.get(i).getMeasurementName().equals(lowerCaseColumnName)) { return i; } } @@ -144,13 +175,14 @@ public int findColumnIndex(String columnName) { * not an ID column */ public int findIdColumnOrder(String columnName) { + final String lowerCaseColumnName = columnName.toLowerCase(); return getIdColumnOrder() .computeIfAbsent( - columnName, + lowerCaseColumnName, colName -> { int columnOrder = 0; for (int i = 0; i < measurementSchemas.size(); i++) { - if (measurementSchemas.get(i).getMeasurementName().equals(columnName) + if (measurementSchemas.get(i).getMeasurementName().equals(lowerCaseColumnName) && columnCategories.get(i) == ColumnCategory.ID) { return columnOrder; } else if (columnCategories.get(i) == ColumnCategory.ID) { @@ -162,7 +194,7 @@ public int findIdColumnOrder(String columnName) { } public IMeasurementSchema findColumnSchema(String columnName) { - final int columnIndex = findColumnIndex(columnName); + final int columnIndex = findColumnIndex(columnName.toLowerCase()); return columnIndex >= 0 ? measurementSchemas.get(columnIndex) : null; } @@ -230,7 +262,7 @@ public static TableSchema deserialize(ByteBuffer buffer, DeserializeConfig conte measurementSchemas.add(measurementSchema); columnCategories.add(ColumnCategory.values()[buffer.getInt()]); } - return new TableSchema(null, measurementSchemas, columnCategories); + return new TableSchema(measurementSchemas, columnCategories); } public String getTableName() { @@ -238,7 +270,7 @@ public String getTableName() { } public void setTableName(String tableName) { - this.tableName = tableName; + this.tableName = tableName.toLowerCase(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java index a354df773..50f6ab6e3 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java @@ -74,30 +74,33 @@ public List getAllTableSchema() throws IOException { public Optional getTableSchemas(String tableName) throws IOException { TsFileMetadata tsFileMetadata = fileReader.readFileMetadata(); Map tableSchemaMap = tsFileMetadata.getTableSchemaMap(); - return Optional.ofNullable(tableSchemaMap.get(tableName)); + return Optional.ofNullable(tableSchemaMap.get(tableName.toLowerCase())); } @TsFileApi public ResultSet query(String tableName, List columnNames, long startTime, long endTime) throws IOException, NoTableException, NoMeasurementException, ReadProcessException { + String lowerCaseTableName = tableName.toLowerCase(); TsFileMetadata tsFileMetadata = fileReader.readFileMetadata(); - TableSchema tableSchema = tsFileMetadata.getTableSchemaMap().get(tableName); + TableSchema tableSchema = tsFileMetadata.getTableSchemaMap().get(lowerCaseTableName); if (tableSchema == null) { throw new NoTableException(tableName); } List dataTypeList = new ArrayList<>(columnNames.size()); + List lowerCaseColumnNames = new ArrayList<>(columnNames.size()); for (String columnName : columnNames) { Map column2IndexMap = tableSchema.buildColumnPosIndex(); - Integer columnIndex = column2IndexMap.get(columnName); + Integer columnIndex = column2IndexMap.get(columnName.toLowerCase()); if (columnIndex == null) { throw new NoMeasurementException(columnName); } + lowerCaseColumnNames.add(columnName.toLowerCase()); dataTypeList.add(tableSchema.getColumnSchemas().get(columnIndex).getType()); } TsBlockReader tsBlockReader = queryExecutor.query( - tableName, - columnNames, + lowerCaseTableName, + lowerCaseColumnNames, new ExpressionTree.TimeBetweenAnd(startTime, endTime), null, null); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java index a8aae2708..d2f77400c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java @@ -38,6 +38,7 @@ import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl; +import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -347,7 +348,7 @@ private boolean checkIsTimeseriesExist(TSRecord record, boolean isAligned) throws WriteProcessException, IOException { // initial ChunkGroupWriter of this device in the TSRecord final IDeviceID deviceID = record.deviceId; - IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID, isAligned); + IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID, isAligned, false); // initial all SeriesWriters of measurements in this TSRecord List measurementSchemas; @@ -411,7 +412,7 @@ private void checkIsTableExistAndSetColumnCategoryList(Tablet tablet) private void checkIsTimeseriesExist(Tablet tablet, boolean isAligned) throws WriteProcessException, IOException { final IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.getDeviceId()); - IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID, isAligned); + IChunkGroupWriter groupWriter = tryToInitialGroupWriter(deviceID, isAligned, false); List schemas = tablet.getSchemas(); if (getSchema().containsDevice(deviceID)) { @@ -495,11 +496,15 @@ private List checkIsAllMeasurementsInGroup( return schemas; } - private IChunkGroupWriter tryToInitialGroupWriter(IDeviceID deviceId, boolean isAligned) { + private IChunkGroupWriter tryToInitialGroupWriter( + IDeviceID deviceId, boolean isAligned, boolean isTableModel) { IChunkGroupWriter groupWriter = groupWriters.get(deviceId); if (groupWriter == null) { if (isAligned) { - groupWriter = new AlignedChunkGroupWriterImpl(deviceId, encryptParam); + groupWriter = + isTableModel + ? new TableChunkGroupWriterImpl(deviceId, encryptParam) + : new AlignedChunkGroupWriterImpl(deviceId, encryptParam); if (!isUnseq) { // Sequence File ((AlignedChunkGroupWriterImpl) groupWriter) .setLastTime(alignedDeviceLastTimeMap.get(deviceId)); @@ -733,7 +738,7 @@ public boolean writeTable(Tablet tablet, List> deviceId for (Pair pair : deviceIdEndIndexPairs) { // get corresponding ChunkGroupWriter and write this Tablet recordCount += - tryToInitialGroupWriter(pair.left, isTableWriteAligned) + tryToInitialGroupWriter(pair.left, isTableWriteAligned, true) .write(tablet, startIndex, pair.right); startIndex = pair.right; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index e50d8a5cb..d6dd6c872 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -64,6 +64,7 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { private long lastTime = Long.MIN_VALUE; private boolean isInitLastTime = false; + private boolean convertColumnNameToLowerCase = false; public AlignedChunkGroupWriterImpl(IDeviceID deviceId) { this.deviceId = deviceId; @@ -100,17 +101,20 @@ public void tryToAddSeriesWriter(IMeasurementSchema measurementSchema) throws IO public ValueChunkWriter tryToAddSeriesWriterInternal(IMeasurementSchema measurementSchema) throws IOException { - ValueChunkWriter valueChunkWriter = - valueChunkWriterMap.get(measurementSchema.getMeasurementName()); + String measurementName = + convertColumnNameToLowerCase + ? measurementSchema.getMeasurementName().toLowerCase() + : measurementSchema.getMeasurementName(); + ValueChunkWriter valueChunkWriter = valueChunkWriterMap.get(measurementName); if (valueChunkWriter == null) { valueChunkWriter = new ValueChunkWriter( - measurementSchema.getMeasurementName(), + measurementName, measurementSchema.getCompressor(), measurementSchema.getType(), measurementSchema.getEncodingType(), measurementSchema.getValueEncoder()); - valueChunkWriterMap.put(measurementSchema.getMeasurementName(), valueChunkWriter); + valueChunkWriterMap.put(measurementName, valueChunkWriter); tryToAddEmptyPageAndData(valueChunkWriter); } return valueChunkWriter; @@ -119,15 +123,19 @@ public ValueChunkWriter tryToAddSeriesWriterInternal(IMeasurementSchema measurem @Override public void tryToAddSeriesWriter(List measurementSchemas) throws IOException { for (IMeasurementSchema schema : measurementSchemas) { - if (!valueChunkWriterMap.containsKey(schema.getMeasurementName())) { + String measurementName = + convertColumnNameToLowerCase + ? schema.getMeasurementName().toLowerCase() + : schema.getMeasurementName(); + if (!valueChunkWriterMap.containsKey(measurementName)) { ValueChunkWriter valueChunkWriter = new ValueChunkWriter( - schema.getMeasurementName(), + measurementName, schema.getCompressor(), schema.getType(), schema.getEncodingType(), schema.getValueEncoder()); - valueChunkWriterMap.put(schema.getMeasurementName(), valueChunkWriter); + valueChunkWriterMap.put(measurementName, valueChunkWriter); tryToAddEmptyPageAndData(valueChunkWriter); } } @@ -138,7 +146,13 @@ public int write(long time, List data) throws WriteProcessException, checkIsHistoryData(time); List emptyValueChunkWriters = new ArrayList<>(); Set existingMeasurements = - data.stream().map(DataPoint::getMeasurementId).collect(Collectors.toSet()); + data.stream() + .map( + dataPoint -> + convertColumnNameToLowerCase + ? dataPoint.getMeasurementId().toLowerCase() + : dataPoint.getMeasurementId()) + .collect(Collectors.toSet()); for (Map.Entry entry : valueChunkWriterMap.entrySet()) { if (!existingMeasurements.contains(entry.getKey())) { emptyValueChunkWriters.add(entry.getValue()); @@ -146,7 +160,11 @@ public int write(long time, List data) throws WriteProcessException, } for (DataPoint point : data) { boolean isNull = point.getValue() == null; - ValueChunkWriter valueChunkWriter = valueChunkWriterMap.get(point.getMeasurementId()); + String measurementId = + convertColumnNameToLowerCase + ? point.getMeasurementId().toLowerCase() + : point.getMeasurementId(); + ValueChunkWriter valueChunkWriter = valueChunkWriterMap.get(measurementId); switch (point.getType()) { case BOOLEAN: valueChunkWriter.write(time, (boolean) point.getValue(), isNull); @@ -201,7 +219,11 @@ public int write(Tablet tablet, int startRowIndex, int endRowIndex) // TODO: should we allow duplicated measurements in a Tablet? Set existingMeasurements = measurementSchemas.stream() - .map(IMeasurementSchema::getMeasurementName) + .map( + schema -> + convertColumnNameToLowerCase + ? schema.getMeasurementName().toLowerCase() + : schema.getMeasurementName()) .collect(Collectors.toSet()); for (Map.Entry entry : valueChunkWriterMap.entrySet()) { if (!existingMeasurements.contains(entry.getKey())) { @@ -413,4 +435,8 @@ public void setLastTime(Long lastTime) { isInitLastTime = true; } } + + public void setConvertColumnNameToLowerCase(boolean convertColumnNameToLowerCase) { + this.convertColumnNameToLowerCase = convertColumnNameToLowerCase; + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java new file mode 100644 index 000000000..3e987b662 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/TableChunkGroupWriterImpl.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.write.chunk; + +import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.file.metadata.IDeviceID; + +public class TableChunkGroupWriterImpl extends AlignedChunkGroupWriterImpl { + + public TableChunkGroupWriterImpl(IDeviceID deviceId) { + super(deviceId); + setConvertColumnNameToLowerCase(true); + } + + public TableChunkGroupWriterImpl(IDeviceID deviceId, EncryptParameter encryptParam) { + super(deviceId, encryptParam); + setConvertColumnNameToLowerCase(true); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index e39d2d06e..515b41bb2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -279,7 +279,7 @@ public void addTimestamp(int rowIndex, long timestamp) { } public void addValue(final String measurementId, final int rowIndex, final Object value) { - int indexOfSchema = measurementIndex.get(measurementId); + int indexOfSchema = getColumnIndexByMeasurement(measurementId); IMeasurementSchema measurementSchema = schemas.get(indexOfSchema); addValueOfDataType(measurementSchema.getType(), rowIndex, indexOfSchema, value); } @@ -349,7 +349,7 @@ private void addValueOfDataType( @TsFileApi public void addValue(int rowIndex, String measurement, int val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -362,7 +362,7 @@ public void addValue(int rowIndex, int columnIndex, int val) { @TsFileApi public void addValue(int rowIndex, String measurement, long val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -375,7 +375,7 @@ public void addValue(int rowIndex, int columnIndex, long val) { @TsFileApi public void addValue(int rowIndex, String measurement, float val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -388,7 +388,7 @@ public void addValue(int rowIndex, int columnIndex, float val) { @TsFileApi public void addValue(int rowIndex, String measurement, double val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -401,7 +401,7 @@ public void addValue(int rowIndex, int columnIndex, double val) { @TsFileApi public void addValue(int rowIndex, String measurement, boolean val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -414,7 +414,7 @@ public void addValue(int rowIndex, int columnIndex, boolean val) { @TsFileApi public void addValue(int rowIndex, String measurement, String val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -427,7 +427,7 @@ public void addValue(int rowIndex, int columnIndex, String val) { @TsFileApi public void addValue(int rowIndex, String measurement, byte[] val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -440,7 +440,7 @@ public void addValue(int rowIndex, int columnIndex, byte[] val) { @TsFileApi public void addValue(int rowIndex, String measurement, LocalDate val) { - Integer columnIndex = measurementIndex.get(measurement); + int columnIndex = getColumnIndexByMeasurement(measurement); addValue(rowIndex, columnIndex, val); } @@ -451,6 +451,17 @@ public void addValue(int rowIndex, int columnIndex, LocalDate val) { updateBitMap(rowIndex, columnIndex, false); } + private int getColumnIndexByMeasurement(String measurement) { + if (measurement == null) { + throw new IllegalArgumentException("measurement should be non null value"); + } + Integer columnIndex = measurementIndex.get(measurement); + if (columnIndex == null) { + throw new IllegalArgumentException("No measurement for " + measurement); + } + return columnIndex; + } + private void updateBitMap(int rowIndex, int columnIndex, boolean mark) { if (bitMaps == null) { initBitMaps(); @@ -1093,7 +1104,7 @@ public Object getValue(int i, int j) { */ public IDeviceID getDeviceID(int i) { String[] idArray = new String[idColumnIndexes.size() + 1]; - idArray[0] = insertTargetName; + idArray[0] = getTableName(); for (int j = 0; j < idColumnIndexes.size(); j++) { final Object value = getValue(i, idColumnIndexes.get(j)); idArray[j + 1] = value != null ? value.toString() : null; @@ -1154,7 +1165,7 @@ public void setDeviceId(String deviceId) { } public String getTableName() { - return insertTargetName; + return insertTargetName == null ? null : insertTargetName.toLowerCase(); } /** @@ -1163,7 +1174,7 @@ public String getTableName() { * @param tableName set the tableName as the insertTargetName */ public void setTableName(String tableName) { - this.insertTargetName = tableName; + this.insertTargetName = tableName.toLowerCase(); } public List getColumnTypes() { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index 102fcd691..80069d3eb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -29,6 +29,7 @@ import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; import org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl; +import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl; import org.apache.tsfile.write.schema.Schema; import org.apache.tsfile.write.writer.TsFileIOWriter; @@ -139,11 +140,15 @@ protected AbstractTableModelTsFileWriter(File file, long chunkGroupSizeThreshold } } - protected IChunkGroupWriter tryToInitialGroupWriter(IDeviceID deviceId, boolean isAligned) { + protected IChunkGroupWriter tryToInitialGroupWriter( + IDeviceID deviceId, boolean isAligned, boolean isTableModel) { IChunkGroupWriter groupWriter = groupWriters.get(deviceId); if (groupWriter == null) { if (isAligned) { - groupWriter = new AlignedChunkGroupWriterImpl(deviceId, encryptParam); + groupWriter = + isTableModel + ? new TableChunkGroupWriterImpl(deviceId, encryptParam) + : new AlignedChunkGroupWriterImpl(deviceId, encryptParam); ((AlignedChunkGroupWriterImpl) groupWriter) .setLastTime(alignedDeviceLastTimeMap.get(deviceId)); } else { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java index 117fdc120..4b517759d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java @@ -66,7 +66,7 @@ public void write(Tablet table) throws IOException, WriteProcessException { for (Pair pair : deviceIdEndIndexPairs) { // get corresponding ChunkGroupWriter and write this Tablet recordCount += - tryToInitialGroupWriter(pair.left, isTableWriteAligned) + tryToInitialGroupWriter(pair.left, isTableWriteAligned, true) .write(table, startIndex, pair.right); startIndex = pair.right; } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java index bf8273bbd..6ab9dad18 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java @@ -100,19 +100,19 @@ public void testQueryTable() throws Exception { try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile); ResultSet resultSet = - tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"), 0, 2); ) { + tsFileReader.query("T1", Arrays.asList("ID1", "ID2", "S2", "S1"), 0, 2); ) { // id1 id2 s2 s1 ResultSetMetadata resultSetMetadata = resultSet.getMetadata(); // Time id1 id2 s2 s1 Assert.assertEquals("Time", resultSetMetadata.getColumnName(1)); Assert.assertEquals(TSDataType.INT64, resultSetMetadata.getColumnType(1)); - Assert.assertEquals("id1", resultSetMetadata.getColumnName(2)); + Assert.assertEquals("ID1", resultSetMetadata.getColumnName(2)); Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(2)); - Assert.assertEquals("id2", resultSetMetadata.getColumnName(3)); + Assert.assertEquals("ID2", resultSetMetadata.getColumnName(3)); Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(3)); - Assert.assertEquals("s2", resultSetMetadata.getColumnName(4)); + Assert.assertEquals("S2", resultSetMetadata.getColumnName(4)); Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(4)); - Assert.assertEquals("s1", resultSetMetadata.getColumnName(5)); + Assert.assertEquals("S1", resultSetMetadata.getColumnName(5)); Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(5)); Assert.assertTrue(resultSet.next()); @@ -177,7 +177,7 @@ public void testQueryTableWithPartialNullValueInChunk() throws Exception { try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile); ResultSet resultSet = - tsFileReader.query("t1", Arrays.asList("id1", "id2", "s2", "s1"), 0, 2); ) { + tsFileReader.query("T1", Arrays.asList("id1", "id2", "S2", "S1"), 0, 2); ) { // id1 id2 s2 s1 ResultSetMetadata resultSetMetadata = resultSet.getMetadata(); // Time id1 id2 s2 s1 @@ -187,9 +187,9 @@ public void testQueryTableWithPartialNullValueInChunk() throws Exception { Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(2)); Assert.assertEquals("id2", resultSetMetadata.getColumnName(3)); Assert.assertEquals(TSDataType.STRING, resultSetMetadata.getColumnType(3)); - Assert.assertEquals("s2", resultSetMetadata.getColumnName(4)); + Assert.assertEquals("S2", resultSetMetadata.getColumnName(4)); Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(4)); - Assert.assertEquals("s1", resultSetMetadata.getColumnName(5)); + Assert.assertEquals("S1", resultSetMetadata.getColumnName(5)); Assert.assertEquals(TSDataType.BOOLEAN, resultSetMetadata.getColumnType(5)); Assert.assertTrue(resultSet.next()); @@ -203,8 +203,8 @@ public void testQueryTableWithPartialNullValueInChunk() throws Exception { Assert.assertEquals(1, resultSet.getLong(1)); Assert.assertEquals("id_field1", resultSet.getString(2)); Assert.assertEquals("id_field2", resultSet.getString(3)); - Assert.assertTrue(resultSet.isNull("s1")); - Assert.assertFalse(resultSet.getBoolean("s2")); + Assert.assertTrue(resultSet.isNull("S1")); + Assert.assertFalse(resultSet.getBoolean("S2")); } } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java index 26aa7894e..bdd2dcb3e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableSchemaTest.java @@ -117,6 +117,7 @@ public void testLogicalTableSchema() throws IOException { final TableSchema deserialized = TableSchema.deserialize(buffer, new DeserializeConfig()); deserialized.setTableName(tableName); assertEquals(tableSchema, deserialized); + assertEquals(measurementSchemaCnt + 2, deserialized.getColumnSchemas().size()); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java index 788685bd3..f43359393 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java @@ -26,7 +26,9 @@ import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.ColumnSchema; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.read.TsFileReader; @@ -58,6 +60,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; public class TsFileWriteApiTest { private final File f = FSFactoryProducer.getFSFactory().getFile("TsFileWriteTest.tsfile"); @@ -849,4 +852,72 @@ public void writeTsFileByFlushingPageDirectly() throws IOException, WriteProcess throw throwable; } } + + @Test + public void writeTreeTsFileWithUpperCaseColumns() throws IOException, WriteProcessException { + setEnv(100 * 1024 * 1024, 10 * 1024); + String d1 = "root.TEST.D1"; + try (TsFileWriter writer = new TsFileWriter(f)) { + writer.registerTimeseries(d1, new MeasurementSchema("MEASUREMENT1", TSDataType.BOOLEAN)); + TSRecord record = new TSRecord(d1, 1); + record.addPoint("MEASUREMENT1", true); + writer.writeRecord(record); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) { + Assert.assertTrue( + reader.getAllDevices().contains(IDeviceID.Factory.DEFAULT_FACTORY.create(d1))); + Assert.assertTrue(reader.getAllMeasurements().containsKey("MEASUREMENT1")); + } + + Tablet tablet = + new Tablet(d1, Arrays.asList(new MeasurementSchema("MEASUREMENT1", TSDataType.BOOLEAN))); + tablet.addTimestamp(0, 0); + tablet.addValue("MEASUREMENT1", 0, true); + try (TsFileWriter writer = new TsFileWriter(f)) { + writer.registerTimeseries(d1, new MeasurementSchema("MEASUREMENT1", TSDataType.BOOLEAN)); + writer.writeTree(tablet); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) { + Assert.assertTrue( + reader.getAllDevices().contains(IDeviceID.Factory.DEFAULT_FACTORY.create(d1))); + Assert.assertTrue(reader.getAllMeasurements().containsKey("MEASUREMENT1")); + } + } + + @Test + public void writeTableTsFileWithUpperCaseColumns() throws IOException, WriteProcessException { + setEnv(100 * 1024 * 1024, 10 * 1024); + Tablet tablet = + new Tablet( + "TABLE1", + Arrays.asList("IdColumn", "MeasurementColumn"), + Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN), + Arrays.asList(Tablet.ColumnCategory.ID, Tablet.ColumnCategory.MEASUREMENT)); + tablet.addTimestamp(0, 0); + tablet.addValue("IdColumn", 0, "id_field"); + tablet.addValue("MeasurementColumn", 0, true); + TableSchema tableSchema = + new TableSchema( + "Table1", + Arrays.asList( + new ColumnSchema("IDCOLUMN", TSDataType.STRING, Tablet.ColumnCategory.ID), + new ColumnSchema( + "MeasurementColumn", TSDataType.BOOLEAN, Tablet.ColumnCategory.MEASUREMENT))); + Assert.assertEquals("table1", tableSchema.getTableName()); + try (TsFileWriter writer = new TsFileWriter(f)) { + writer.registerTableSchema(tableSchema); + writer.writeTable(tablet); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(f.getPath())) { + Map tableSchemaMap = reader.readFileMetadata().getTableSchemaMap(); + TableSchema tableSchemaInTsFile = tableSchemaMap.get("table1"); + Assert.assertNotNull(tableSchemaInTsFile); + for (IMeasurementSchema columnSchema : tableSchemaInTsFile.getColumnSchemas()) { + Assert.assertEquals( + columnSchema.getMeasurementName().toLowerCase(), columnSchema.getMeasurementName()); + } + Assert.assertTrue(reader.getAllMeasurements().containsKey("measurementcolumn")); + } + } }