Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Correct the retained size calculation for BinaryColumn and BinaryColumnBuilder #374

Merged
merged 2 commits into from
Jan 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,18 @@ public static long sizeOf(double[] arr) {
: alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long) Double.BYTES * arr.length);
}

public static long sizeOf(Accountable[] arr) {
if (arr == null) {
return 0;
} else {
long size = shallowSizeOf(arr);
for (Accountable obj : arr) {
size += obj != null ? obj.ramBytesUsed() : 0;
}
return size;
}
}

/** Returns the size in bytes of the String[] object. */
public static long sizeOf(String[] arr) {
long size = shallowSizeOf(arr);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ public int getSerializedSize() {
}

private int getSerializedSize(IDeviceID deviceID) {
// TODO: add an interface in IDeviceID
int length = deviceID.serializedSize();
return Byte.BYTES + ReadWriteForEncodingUtils.varIntSize(length) + length;
}
Expand All @@ -73,7 +72,6 @@ public static ChunkGroupHeader deserializeFrom(
}
}

// TODO: add an interface in IDeviceID
final IDeviceID deviceID = deserializeDeviceID(inputStream, versionNumber);
return new ChunkGroupHeader(deviceID);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,6 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List<TSD
valueColumnBuilders = new ColumnBuilder[types.size()];

for (int i = 0; i < valueColumnBuilders.length; i++) {
// TODO use Type interface to encapsulate createColumnBuilder to each concrete type class
// instead of switch-case
switch (types.get(i)) {
case BOOLEAN:
valueColumnBuilders[i] =
Expand Down Expand Up @@ -176,8 +174,6 @@ public void buildValueColumnBuilders(List<TSDataType> types) {
valueColumnBuilders = new ColumnBuilder[types.size()];
int initialExpectedEntries = timeColumnBuilder.getPositionCount();
for (int i = 0; i < valueColumnBuilders.length; i++) {
// TODO use Type interface to encapsulate createColumnBuilder to each concrete type class
// instead of switch-case
switch (types.get(i)) {
case BOOLEAN:
valueColumnBuilders[i] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkArrayRange;
import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkReadablePosition;
import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRegion;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObjectArray;

public class BinaryColumn implements Column {

Expand Down Expand Up @@ -75,9 +75,7 @@ public BinaryColumn(int positionCount, Optional<boolean[]> valueIsNull, Binary[]
}
this.valueIsNull = valueIsNull;

// TODO we need to sum up all the Binary's retainedSize here
retainedSizeInBytes =
INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOfObjectArray(positionCount);
retainedSizeInBytes = INSTANCE_SIZE + sizeOfBooleanArray(positionCount) + sizeOf(values);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@

import static java.lang.Math.max;
import static org.apache.tsfile.read.common.block.column.ColumnUtil.calculateBlockResetSize;
import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOf;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOf;

public class BinaryColumnBuilder implements ColumnBuilder {
Expand Down Expand Up @@ -129,7 +128,6 @@ public TSDataType getDataType() {

@Override
public long getRetainedSizeInBytes() {
// TODO we need to sum up all the Binary's retainedSize here
long size = INSTANCE_SIZE + arraysRetainedSizeInBytes;
if (columnBuilderStatus != null) {
size += ColumnBuilderStatus.INSTANCE_SIZE;
Expand All @@ -139,7 +137,6 @@ public long getRetainedSizeInBytes() {

@Override
public ColumnBuilder newColumnBuilderLike(ColumnBuilderStatus columnBuilderStatus) {
// TODO we should take retain size into account here
return new BinaryColumnBuilder(columnBuilderStatus, calculateBlockResetSize(positionCount));
}

Expand All @@ -158,6 +155,6 @@ private void growCapacity() {
}

private void updateArraysDataSize() {
arraysRetainedSizeInBytes = sizeOf(valueIsNull) + shallowSizeOf(values);
arraysRetainedSizeInBytes = sizeOf(valueIsNull) + sizeOf(values);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@ public boolean isNull(int position) {

@Override
public boolean[] isNull() {
// todo
return null;
throw new UnsupportedOperationException("isNull is not supported for TimeColumn");
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ public TsBlock deserialize(ByteBuffer byteBuffer) {
}

// Time column.
// TODO: a TimeColumn will be deserialized as a LongColumn
Column timeColumn =
ColumnEncoderFactory.get(columnEncodings.get(0))
.readColumn(byteBuffer, TSDataType.INT64, positionCount);
Expand Down
Loading