Skip to content

Commit

Permalink
feat: add index mem cost for TVList
Browse files Browse the repository at this point in the history
  • Loading branch information
@shizy818
shizy818 committed Jan 14, 2025
1 parent 11f376c commit 736e596
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 25 deletions.
2 changes: 1 addition & 1 deletion ...src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ public String toString() {
continue;
}
size++;
long currentTime = list.getTime(i);
long currentTime = tvList.getTime(i);
if (currentTime < minTime) {
firstTvPair = tvList.getTimeValuePair(i);
minTime = currentTime;
Expand Down
8 changes: 6 additions & 2 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -908,12 +908,13 @@ public static long alignedTvListArrayMemCost(

int measurementColumnNum = 0;
long size = 0;
// value array mem size
// value & bitmap array mem size
for (int i = 0; i < types.length; i++) {
TSDataType type = types[i];
if (type != null
&& (columnCategories == null || columnCategories[i] == TsTableColumnCategory.FIELD)) {
size += (long) ARRAY_SIZE * (long) type.getDataTypeSize();
size += (long) ARRAY_SIZE / 8 + 1;
measurementColumnNum++;
}
}
Expand All @@ -940,10 +941,11 @@ public static long alignedTvListArrayMemCost(
*/
public static long alignedTvListArrayMemCost(List<TSDataType> types) {
long size = 0;
// value array mem size
// value & bitmap array mem size
for (TSDataType type : types) {
if (type != null) {
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1;
}
}
// size is 0 when all types are null
Expand Down Expand Up @@ -971,6 +973,8 @@ public static long valueListArrayMemCost(TSDataType type) {
long size = 0;
// value array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
// bitmap array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1;
// array headers mem size
size += NUM_BYTES_ARRAY_HEADER;
// Object references size in ArrayList
Expand Down
5 changes: 4 additions & 1 deletion iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,16 @@ public static TVList newList(TSDataType dataType) {
return null;
}

// TODO: memory cost for indices and bitmap
public static long tvListArrayMemCost(TSDataType type) {
long size = 0;
// time array mem size
size += PrimitiveArrayManager.ARRAY_SIZE * 8L;
// value array mem size
size += PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
// index array mem size
size += PrimitiveArrayManager.ARRAY_SIZE * 4L;
// bimap array mem size
size += PrimitiveArrayManager.ARRAY_SIZE / 8 + 1L;
// two array headers mem size
size += NUM_BYTES_ARRAY_HEADER * 2L;
// Object references size in ArrayList
Expand Down
16 changes: 13 additions & 3 deletions ...est/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest;
import org.apache.iotdb.db.utils.MathUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;

import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
Expand All @@ -57,7 +58,9 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

public class PrimitiveMemTableTest {
Expand Down Expand Up @@ -96,16 +99,23 @@ public void setUp() {
}

@Test
public void memSeriesSortIteratorTest() throws IOException {
public void memSeriesSortIteratorTest() throws IOException, QueryProcessException {
TSDataType dataType = TSDataType.INT32;
WritableMemChunk series =
new WritableMemChunk(new MeasurementSchema("s1", dataType, TSEncoding.PLAIN));
int count = 1000;
for (int i = 0; i < count; i++) {
series.writeNonAlignedPoint(i, i);
}
IPointReader it =
series.getSortedTvListForQuery().buildTsBlock().getTsBlockSingleColumnIterator();
Map<TVList, Integer> tvListQueryMap = new HashMap<>();
for (TVList tvList : series.getSortedList()) {
tvListQueryMap.put(tvList, tvList.rowCount());
}
tvListQueryMap.put(series.getWorkingTVList(), series.getWorkingTVList().rowCount());
ReadOnlyMemChunk readableChunk =
new ReadOnlyMemChunk(
new QueryContext(), "s1", dataType, TSEncoding.PLAIN, tvListQueryMap, null, null);
IPointReader it = readableChunk.getPointReader();
int i = 0;
while (it.hasNextTimeValuePair()) {
Assert.assertEquals(i, it.nextTimeValuePair().getTimestamp());
Expand Down
36 changes: 18 additions & 18 deletions .../test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -685,21 +685,21 @@ public void alignedTvListRamCostTest()
true,
new long[5]);
IMemTable memTable = processor.getWorkMemTable();
Assert.assertEquals(1596808, memTable.getTVListsRamCost());
Assert.assertEquals(1623808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(100, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(1596808, memTable.getTVListsRamCost());
Assert.assertEquals(1623808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(200, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(1596808, memTable.getTVListsRamCost());
Assert.assertEquals(1623808, memTable.getTVListsRamCost());
Assert.assertEquals(90000, memTable.getTotalPointsNum());
Assert.assertEquals(720360, memTable.memSize());
// Test records
Expand All @@ -708,7 +708,7 @@ public void alignedTvListRamCostTest()
record.addTuple(DataPoint.getDataPoint(dataType, measurementId, String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]);
}
Assert.assertEquals(1598424, memTable.getTVListsRamCost());
Assert.assertEquals(1625954, memTable.getTVListsRamCost());
Assert.assertEquals(90100, memTable.getTotalPointsNum());
Assert.assertEquals(721560, memTable.memSize());
}
Expand Down Expand Up @@ -736,56 +736,56 @@ public void alignedTvListRamCostTest2()
true,
new long[5]);
IMemTable memTable = processor.getWorkMemTable();
Assert.assertEquals(1596808, memTable.getTVListsRamCost());
Assert.assertEquals(1623808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNodeFors3000ToS6000(0, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192808, memTable.getTVListsRamCost());
Assert.assertEquals(3246808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(100, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192808, memTable.getTVListsRamCost());
Assert.assertEquals(3246808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNodeFors3000ToS6000(100, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192808, memTable.getTVListsRamCost());
Assert.assertEquals(3246808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(200, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192808, memTable.getTVListsRamCost());
Assert.assertEquals(3246808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNodeFors3000ToS6000(200, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192808, memTable.getTVListsRamCost());
Assert.assertEquals(3246808, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(300, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(6385616, memTable.getTVListsRamCost());
Assert.assertEquals(6493616, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNodeFors3000ToS6000(300, true),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(6385616, memTable.getTVListsRamCost());
Assert.assertEquals(6493616, memTable.getTVListsRamCost());

Assert.assertEquals(240000, memTable.getTotalPointsNum());
Assert.assertEquals(1920960, memTable.memSize());
Expand All @@ -795,14 +795,14 @@ public void alignedTvListRamCostTest2()
record.addTuple(DataPoint.getDataPoint(dataType, measurementId, String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]);
}
Assert.assertEquals(6387232, memTable.getTVListsRamCost());
Assert.assertEquals(6495762, memTable.getTVListsRamCost());
// Test records
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(deviceId, i);
record.addTuple(DataPoint.getDataPoint(dataType, "s1", String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]);
}
Assert.assertEquals(6388848, memTable.getTVListsRamCost());
Assert.assertEquals(6497908, memTable.getTVListsRamCost());
Assert.assertEquals(240200, memTable.getTotalPointsNum());
Assert.assertEquals(1923360, memTable.memSize());
}
Expand Down Expand Up @@ -830,21 +830,21 @@ public void nonAlignedTvListRamCostTest()
true,
new long[5]);
IMemTable memTable = processor.getWorkMemTable();
Assert.assertEquals(3192000, memTable.getTVListsRamCost());
Assert.assertEquals(3987000, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(100, false),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192000, memTable.getTVListsRamCost());
Assert.assertEquals(3987000, memTable.getTVListsRamCost());
processor.insertTablet(
genInsertTableNode(200, false),
Collections.singletonList(new int[] {0, 10}),
new TSStatus[10],
true,
new long[5]);
Assert.assertEquals(3192000, memTable.getTVListsRamCost());
Assert.assertEquals(3987000, memTable.getTVListsRamCost());
Assert.assertEquals(90000, memTable.getTotalPointsNum());
Assert.assertEquals(1440000, memTable.memSize());
// Test records
Expand All @@ -853,7 +853,7 @@ public void nonAlignedTvListRamCostTest()
record.addTuple(DataPoint.getDataPoint(dataType, measurementId, String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[5]);
}
Assert.assertEquals(3193616, memTable.getTVListsRamCost());
Assert.assertEquals(3989146, memTable.getTVListsRamCost());
Assert.assertEquals(90100, memTable.getTotalPointsNum());
Assert.assertEquals(1441200, memTable.memSize());
}
Expand Down

0 comments on commit 736e596

Please sign in to comment.