From e891e0ba312d2cb47b642500cf29313e50470d7e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:32:16 +0800 Subject: [PATCH 1/9] Improve pipe unit tests --- .../event/PipeDataTypeTransformerTest.java | 107 ++++--- .../db/pipe/pattern/IoTDBTreePatternTest.java | 272 +++++++++++------- .../thrift/request/PipeRequestTypeTest.java | 102 +++++-- 3 files changed, 303 insertions(+), 178 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java index a2249a35e50e6..9369daeb06f68 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeDataTypeTransformerTest.java @@ -24,60 +24,81 @@ import org.apache.tsfile.enums.TSDataType; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class PipeDataTypeTransformerTest { - private List nullTsDataTypeList; - private List tsDataTypeList; - private List nullTypeList; - private List typeList; - @Before - public void setUp() { - createDataTypeList(); - } + @Test + public void testTransformToPipeDataType() { + for (final DataTypeCase dataTypeCase : dataTypeCases()) { + Assert.assertEquals( + dataTypeCase.pipeDataType, + PipeDataTypeTransformer.transformToPipeDataType(dataTypeCase.tsDataType)); + } - public void createDataTypeList() { - nullTsDataTypeList = null; - tsDataTypeList = new ArrayList<>(); - tsDataTypeList.add(TSDataType.INT32); - tsDataTypeList.add(TSDataType.INT64); - tsDataTypeList.add(TSDataType.FLOAT); - tsDataTypeList.add(TSDataType.DOUBLE); - tsDataTypeList.add(TSDataType.BOOLEAN); - tsDataTypeList.add(TSDataType.TEXT); - tsDataTypeList.add(TSDataType.TIMESTAMP); - tsDataTypeList.add(TSDataType.DATE); - tsDataTypeList.add(TSDataType.BLOB); - tsDataTypeList.add(TSDataType.STRING); - tsDataTypeList.add(null); + Assert.assertNull(PipeDataTypeTransformer.transformToPipeDataType(null)); + } - nullTypeList = null; - typeList = new ArrayList<>(); - typeList.add(Type.INT32); - typeList.add(Type.INT64); - typeList.add(Type.FLOAT); - typeList.add(Type.DOUBLE); - typeList.add(Type.BOOLEAN); - typeList.add(Type.TEXT); - typeList.add(Type.TIMESTAMP); - typeList.add(Type.DATE); - typeList.add(Type.BLOB); - typeList.add(Type.STRING); - typeList.add(null); + @Test + public void testUnsupportedDataType() { + for (final TSDataType unsupportedDataType : + Arrays.asList(TSDataType.VECTOR, TSDataType.UNKNOWN)) { + try { + PipeDataTypeTransformer.transformToPipeDataType(unsupportedDataType); + Assert.fail("Should reject unsupported TSDataType " + unsupportedDataType); + } catch (final IllegalArgumentException ignored) { + // Expected exception + } + } } @Test - public void testDataTypeTransformer() { - final List nullResultList = - PipeDataTypeTransformer.transformToPipeDataTypeList(nullTsDataTypeList); - final List resultList = - PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypeList); - Assert.assertEquals(nullResultList, nullTypeList); - Assert.assertEquals(resultList, typeList); + public void testTransformToPipeDataTypeList() { + Assert.assertNull(PipeDataTypeTransformer.transformToPipeDataTypeList(null)); + Assert.assertEquals( + Collections.emptyList(), + PipeDataTypeTransformer.transformToPipeDataTypeList(Collections.emptyList())); + + final List tsDataTypes = new ArrayList<>(); + final List pipeDataTypes = new ArrayList<>(); + for (final DataTypeCase dataTypeCase : dataTypeCases()) { + tsDataTypes.add(dataTypeCase.tsDataType); + pipeDataTypes.add(dataTypeCase.pipeDataType); + } + tsDataTypes.add(null); + pipeDataTypes.add(null); + + Assert.assertEquals( + pipeDataTypes, PipeDataTypeTransformer.transformToPipeDataTypeList(tsDataTypes)); + } + + private static List dataTypeCases() { + return Arrays.asList( + new DataTypeCase(TSDataType.BOOLEAN, Type.BOOLEAN), + new DataTypeCase(TSDataType.INT32, Type.INT32), + new DataTypeCase(TSDataType.INT64, Type.INT64), + new DataTypeCase(TSDataType.FLOAT, Type.FLOAT), + new DataTypeCase(TSDataType.DOUBLE, Type.DOUBLE), + new DataTypeCase(TSDataType.TEXT, Type.TEXT), + new DataTypeCase(TSDataType.TIMESTAMP, Type.TIMESTAMP), + new DataTypeCase(TSDataType.DATE, Type.DATE), + new DataTypeCase(TSDataType.BLOB, Type.BLOB), + new DataTypeCase(TSDataType.STRING, Type.STRING)); + } + + private static class DataTypeCase { + + private final TSDataType tsDataType; + private final Type pipeDataType; + + private DataTypeCase(final TSDataType tsDataType, final Type pipeDataType) { + this.tsDataType = tsDataType; + this.pipeDataType = pipeDataType; + } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/IoTDBTreePatternTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/IoTDBTreePatternTest.java index 6b17bdee6d3e4..9386da4f6ce58 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/IoTDBTreePatternTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/IoTDBTreePatternTest.java @@ -27,127 +27,189 @@ import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.IntStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Predicate; import static org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern.applyReversedIndexesOnList; public class IoTDBTreePatternTest { + private static final String DB = "root.db"; + private static final IDeviceID DEVICE = new StringArrayDeviceID("root.db.d1"); + private static final String MEASUREMENT = "s1"; + @Test - public void testIotdbPipePattern() { - // Test legal and illegal pattern - final String[] legalPatterns = { - "root", "root.db", "root.db.d1.s", "root.db.`1`", "root.*.d.*s.s", - }; - final String[] illegalPatterns = { - "root.", "roo", "", "root..", "root./", - }; - for (final String s : legalPatterns) { - Assert.assertTrue(new IoTDBTreePattern(s).isLegal()); - } - for (final String t : illegalPatterns) { - try { - Assert.assertFalse(new IoTDBTreePattern(t).isLegal()); - } catch (final Exception e) { - Assert.assertTrue(e instanceof PipeException); - } - } + public void testLegalPattern() { + assertLegal("root", "root.db", "root.db.d1.s", "root.db.`1`", "root.*.d.*s.s", null); + assertIllegalOrInvalid("root.", "roo", "", "root..", "root./"); + } - // Test pattern cover db - final String db = "root.db"; - final String[] patternsCoverDb = { - "root.**", "root.db.**", "root.*db*.**", - }; - final String[] patternsNotCoverDb = { - "root.db", "root.*", "root.*.*", "root.db.*.**", "root.db.d1", "root.**.db.**", - }; - for (final String s : patternsCoverDb) { - Assert.assertTrue(new IoTDBTreePattern(s).coversDb(db)); - } - for (final String t : patternsNotCoverDb) { - Assert.assertFalse(new IoTDBTreePattern(t).coversDb(db)); - } + @Test + public void testRootPattern() { + Assert.assertTrue(new IoTDBTreePattern(null).isRoot()); + Assert.assertTrue(new IoTDBTreePattern("root.**").isRoot()); + Assert.assertFalse(new IoTDBTreePattern("root").isRoot()); + } - final IDeviceID device = new StringArrayDeviceID("root.db.d1"); - - // Test pattern cover device - final String[] patternsCoverDevice = { - "root.**", "root.db.**", "root.*.*.*", "root.db.d1.*", "root.*db*.*d*.*", "root.**.*1.*", - }; - final String[] patternsNotCoverDevice = { - "root.*", "root.*.*", "root.db.d1", "root.db.d2.*", "root.**.d2.**", - }; - for (final String s : patternsCoverDevice) { - Assert.assertTrue(new IoTDBTreePattern(s).coversDevice(device)); - } - for (String t : patternsNotCoverDevice) { - Assert.assertFalse(new IoTDBTreePattern(t).coversDevice(device)); - } + @Test + public void testCoversDb() { + assertPatternResult( + true, pattern -> pattern.coversDb(DB), "root.**", "root.db.**", "root.*db*.**"); + assertPatternResult( + false, + pattern -> pattern.coversDb(DB), + "root.db", + "root.*", + "root.*.*", + "root.db.*.**", + "root.db.d1", + "root.**.db.**"); + } - // Test pattern may overlap with device - final String[] patternsOverlapWithDevice = { - "root.db.**", "root.db.d1", "root.db.d1.*", "root.db.d1.s1", "root.**.d2.**", "root.*.d*.**", - }; - final String[] patternsNotOverlapWithDevice = { - "root.db.d2.**", "root.db2.d1.**", "root.db.db.d1.**", - }; - final String[] patternsFalsePositiveOverLap = {"root.**.d2.**"}; - for (final String s : patternsOverlapWithDevice) { - Assert.assertTrue(new IoTDBTreePattern(s).mayOverlapWithDevice(device)); - } - for (final String t : patternsNotOverlapWithDevice) { - Assert.assertFalse(new IoTDBTreePattern(t).mayOverlapWithDevice(device)); - } - for (final String t : patternsFalsePositiveOverLap) { - Assert.assertTrue(new IoTDBTreePattern(t).mayOverlapWithDevice(device)); - Assert.assertFalse(new IoTDBTreePattern(t).overlapWithDevice(device)); - } + @Test + public void testMayOverlapWithDb() { + assertPatternResult( + true, + pattern -> pattern.mayOverlapWithDb(DB), + "root.**", + "root.db.**", + "root.db.d1", + "root.*.d1"); + assertPatternResult( + false, + pattern -> pattern.mayOverlapWithDb(DB), + "root.other.**", + "root.other.d1", + "root.db2.d1"); + } - // Test pattern match measurement - final String measurement = "s1"; - final String[] patternsMatchMeasurement = { - "root.db.d1.s1", "root.db.d1.*", - }; - final String[] patternsNotMatchMeasurement = { - "root.db.d1", "root.db.d1", "root.db.d1.*.*", - }; - for (final String s : patternsMatchMeasurement) { - Assert.assertTrue(new IoTDBTreePattern(s).matchesMeasurement(device, measurement)); - } - for (final String t : patternsNotMatchMeasurement) { - Assert.assertFalse(new IoTDBTreePattern(t).matchesMeasurement(device, measurement)); - } + @Test + public void testCoversDevice() { + assertPatternResult( + true, + pattern -> pattern.coversDevice(DEVICE), + "root.**", + "root.db.**", + "root.*.*.*", + "root.db.d1.*", + "root.*db*.*d*.*", + "root.**.*1.*"); + assertPatternResult( + false, + pattern -> pattern.coversDevice(DEVICE), + "root.*", + "root.*.*", + "root.db.d1", + "root.db.d2.*", + "root.**.d2.**"); + } + + @Test + public void testOverlapsDevice() { + assertPatternResult( + true, + pattern -> pattern.mayOverlapWithDevice(DEVICE), + "root.db.**", + "root.db.d1", + "root.db.d1.*", + "root.db.d1.s1", + "root.**.d2.**", + "root.*.d*.**"); + assertPatternResult( + false, + pattern -> pattern.mayOverlapWithDevice(DEVICE), + "root.db.d2.**", + "root.db2.d1.**", + "root.db.db.d1.**"); + + final IoTDBTreePattern falsePositivePattern = new IoTDBTreePattern("root.**.d2.**"); + Assert.assertTrue(falsePositivePattern.mayOverlapWithDevice(DEVICE)); + Assert.assertFalse(falsePositivePattern.overlapWithDevice(DEVICE)); + } + + @Test + public void testMatchesMeasurement() { + assertPatternResult( + true, + pattern -> pattern.matchesMeasurement(DEVICE, MEASUREMENT), + "root.db.d1.s1", + "root.db.d1.*"); + assertPatternResult( + false, + pattern -> pattern.matchesMeasurement(DEVICE, MEASUREMENT), + "root.db.d1", + "root.db.d1.*.*", + "root.db.d2.s1"); + + final IoTDBTreePattern matchAllMeasurements = new IoTDBTreePattern("root.db.d1.*"); + Assert.assertFalse(matchAllMeasurements.matchesMeasurement(DEVICE, null)); + Assert.assertFalse(matchAllMeasurements.matchesMeasurement(DEVICE, "")); + } + + @Test + public void testSchemaPatternHelpers() { + final IoTDBTreePattern prefixPattern = new IoTDBTreePattern("root.db.**"); + Assert.assertTrue(prefixPattern.matchPrefixPath("root.db")); + Assert.assertFalse(prefixPattern.matchPrefixPath("root.other")); + Assert.assertTrue(prefixPattern.matchTailNode("any")); + Assert.assertTrue(prefixPattern.isPrefixOrFullPath()); + Assert.assertTrue(prefixPattern.mayMatchMultipleTimeSeriesInOneDevice()); + + final IoTDBTreePattern fullPathPattern = new IoTDBTreePattern("root.db.d1.s1"); + Assert.assertTrue(fullPathPattern.matchTailNode("s1")); + Assert.assertFalse(fullPathPattern.matchTailNode("s2")); + Assert.assertTrue(fullPathPattern.isPrefixOrFullPath()); + Assert.assertFalse(fullPathPattern.mayMatchMultipleTimeSeriesInOneDevice()); + + final IoTDBTreePattern wildcardMiddlePattern = new IoTDBTreePattern("root.*.**"); + Assert.assertFalse(wildcardMiddlePattern.isPrefixOrFullPath()); + } + + @Test + public void testMatchDevice() { + assertPatternResult( + true, pattern -> pattern.matchDevice("root.db.d1"), "root.db.d1.*", "root.db.**"); + assertPatternResult( + false, pattern -> pattern.matchDevice("root.db.d1"), "root.db.d2.*", "root.db2.**"); } @Test public void testApplyReversedIndexes() { - final int elementNum = 10_000_000; - final int filteredNum = elementNum / 10; - final Random random = new Random(); - final List originalList = - IntStream.range(0, elementNum).boxed().collect(Collectors.toList()); - List filteredIndexes = new ArrayList<>(filteredNum); - for (int i = 0; i < filteredNum; i++) { - filteredIndexes.add(random.nextInt(elementNum)); + Assert.assertEquals( + Arrays.asList(0, 2, 4), + applyReversedIndexesOnList(Arrays.asList(1, 3), Arrays.asList(0, 1, 2, 3, 4))); + Assert.assertEquals( + Arrays.asList("a", "b"), + applyReversedIndexesOnList(Collections.emptyList(), Arrays.asList("a", "b"))); + Assert.assertEquals( + Collections.emptyList(), + applyReversedIndexesOnList(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2))); + Assert.assertNull(applyReversedIndexesOnList(Collections.singletonList(0), null)); + } + + private static void assertLegal(final String... patterns) { + for (final String pattern : patterns) { + Assert.assertTrue(String.valueOf(pattern), new IoTDBTreePattern(pattern).isLegal()); } - filteredIndexes = filteredIndexes.stream().sorted().distinct().collect(Collectors.toList()); - - final long start = System.currentTimeMillis(); - final List appliedList = applyReversedIndexesOnList(filteredIndexes, originalList); - System.out.println(System.currentTimeMillis() - start); - final Set appliedSet = new HashSet<>(appliedList); - for (Integer filteredIndex : filteredIndexes) { - if (appliedSet.contains(filteredIndex)) { - System.out.println("Incorrect implementation"); - System.exit(-1); + } + + private static void assertIllegalOrInvalid(final String... patterns) { + for (final String pattern : patterns) { + try { + Assert.assertFalse(String.valueOf(pattern), new IoTDBTreePattern(pattern).isLegal()); + } catch (final Exception e) { + Assert.assertTrue(e instanceof PipeException); } } - Assert.assertEquals(null, applyReversedIndexesOnList(filteredIndexes, null)); + } + + private static void assertPatternResult( + final boolean expected, + final Predicate assertion, + final String... patterns) { + for (final String pattern : patterns) { + Assert.assertEquals(pattern, expected, assertion.test(new IoTDBTreePattern(pattern))); + } } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeRequestTypeTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeRequestTypeTest.java index c2e0ba949f984..bc4ec1c286ea7 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeRequestTypeTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeRequestTypeTest.java @@ -22,40 +22,82 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + public class PipeRequestTypeTest { @Test - public void testAllV13RequestTypesAreRecognized() { - assertV13RequestType((short) 0, PipeRequestType.HANDSHAKE_CONFIGNODE_V1); - assertV13RequestType((short) 1, PipeRequestType.HANDSHAKE_DATANODE_V1); - assertV13RequestType((short) 50, PipeRequestType.HANDSHAKE_CONFIGNODE_V2); - assertV13RequestType((short) 51, PipeRequestType.HANDSHAKE_DATANODE_V2); - - assertV13RequestType((short) 2, PipeRequestType.TRANSFER_TABLET_INSERT_NODE); - assertV13RequestType((short) 3, PipeRequestType.TRANSFER_TABLET_RAW); - assertV13RequestType((short) 4, PipeRequestType.TRANSFER_TS_FILE_PIECE); - assertV13RequestType((short) 5, PipeRequestType.TRANSFER_TS_FILE_SEAL); - assertV13RequestType((short) 6, PipeRequestType.TRANSFER_TABLET_BATCH); - assertV13RequestType((short) 7, PipeRequestType.TRANSFER_TABLET_BINARY); - assertV13RequestType((short) 8, PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD); - assertV13RequestType((short) 9, PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD); - - // 1.3 named this request TRANSFER_SCHEMA_PLAN. 2.0 keeps the same wire type. - assertV13RequestType((short) 100, PipeRequestType.TRANSFER_PLAN_NODE); - assertV13RequestType((short) 101, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE); - assertV13RequestType((short) 102, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL); - - assertV13RequestType((short) 200, PipeRequestType.TRANSFER_CONFIG_PLAN); - assertV13RequestType((short) 201, PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE); - assertV13RequestType((short) 202, PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL); - - assertV13RequestType((short) 300, PipeRequestType.TRANSFER_COMPRESSED); - assertV13RequestType((short) 400, PipeRequestType.TRANSFER_SLICE); + public void testAllRequestTypesAreRecognized() { + final Set coveredRequestTypes = new HashSet<>(); + final Set usedWireTypes = new HashSet<>(); + + for (final RequestTypeCase requestTypeCase : requestTypeCases()) { + Assert.assertEquals(requestTypeCase.wireType, requestTypeCase.requestType.getType()); + Assert.assertTrue( + "duplicated request type case: " + requestTypeCase.requestType, + coveredRequestTypes.add(requestTypeCase.requestType)); + Assert.assertTrue( + "duplicated wire type: " + requestTypeCase.wireType, + usedWireTypes.add(requestTypeCase.wireType)); + Assert.assertTrue(PipeRequestType.isValidatedRequestType(requestTypeCase.wireType)); + Assert.assertEquals( + requestTypeCase.requestType, PipeRequestType.valueOf(requestTypeCase.wireType)); + } + + Assert.assertEquals(PipeRequestType.values().length, coveredRequestTypes.size()); + } + + @Test + public void testInvalidRequestTypesAreRejected() { + for (final short invalidType : + new short[] { + Short.MIN_VALUE, -1, 14, 49, 52, 99, 103, 199, 203, 299, 301, 399, 401, Short.MAX_VALUE + }) { + Assert.assertFalse(PipeRequestType.isValidatedRequestType(invalidType)); + Assert.assertNull(PipeRequestType.valueOf(invalidType)); + } + } + + private static List requestTypeCases() { + return Arrays.asList( + new RequestTypeCase((short) 0, PipeRequestType.HANDSHAKE_CONFIGNODE_V1), + new RequestTypeCase((short) 1, PipeRequestType.HANDSHAKE_DATANODE_V1), + new RequestTypeCase((short) 50, PipeRequestType.HANDSHAKE_CONFIGNODE_V2), + new RequestTypeCase((short) 51, PipeRequestType.HANDSHAKE_DATANODE_V2), + new RequestTypeCase((short) 2, PipeRequestType.TRANSFER_TABLET_INSERT_NODE), + new RequestTypeCase((short) 3, PipeRequestType.TRANSFER_TABLET_RAW), + new RequestTypeCase((short) 4, PipeRequestType.TRANSFER_TS_FILE_PIECE), + new RequestTypeCase((short) 5, PipeRequestType.TRANSFER_TS_FILE_SEAL), + new RequestTypeCase((short) 6, PipeRequestType.TRANSFER_TABLET_BATCH), + new RequestTypeCase((short) 7, PipeRequestType.TRANSFER_TABLET_BINARY), + new RequestTypeCase((short) 8, PipeRequestType.TRANSFER_TS_FILE_PIECE_WITH_MOD), + new RequestTypeCase((short) 9, PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD), + new RequestTypeCase((short) 10, PipeRequestType.TRANSFER_TABLET_INSERT_NODE_V2), + new RequestTypeCase((short) 11, PipeRequestType.TRANSFER_TABLET_RAW_V2), + new RequestTypeCase((short) 12, PipeRequestType.TRANSFER_TABLET_BINARY_V2), + new RequestTypeCase((short) 13, PipeRequestType.TRANSFER_TABLET_BATCH_V2), + // 1.3 named this request TRANSFER_SCHEMA_PLAN. 2.0 keeps the same wire type. + new RequestTypeCase((short) 100, PipeRequestType.TRANSFER_PLAN_NODE), + new RequestTypeCase((short) 101, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_PIECE), + new RequestTypeCase((short) 102, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL), + new RequestTypeCase((short) 200, PipeRequestType.TRANSFER_CONFIG_PLAN), + new RequestTypeCase((short) 201, PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_PIECE), + new RequestTypeCase((short) 202, PipeRequestType.TRANSFER_CONFIG_SNAPSHOT_SEAL), + new RequestTypeCase((short) 300, PipeRequestType.TRANSFER_COMPRESSED), + new RequestTypeCase((short) 400, PipeRequestType.TRANSFER_SLICE)); } - private static void assertV13RequestType( - final short type, final PipeRequestType expectedRequestType) { - Assert.assertTrue(PipeRequestType.isValidatedRequestType(type)); - Assert.assertEquals(expectedRequestType, PipeRequestType.valueOf(type)); + private static class RequestTypeCase { + + private final short wireType; + private final PipeRequestType requestType; + + private RequestTypeCase(final short wireType, final PipeRequestType requestType) { + this.wireType = wireType; + this.requestType = requestType; + } } } From a4203a176a5f2757117745457754775ad9dab604 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:24:03 +0800 Subject: [PATCH 2/9] Improve pipe transfer request tests --- .../PipeTransferSliceReqBuilderTest.java | 87 ++++++++-- .../PipeTransferCompressedReqTest.java | 71 ++++++-- .../request/PipeTransferFilePieceReqTest.java | 103 +++++++++++ .../PipeTransferFileSealReqV1Test.java | 96 +++++++++++ .../PipeTransferFileSealReqV2Test.java | 161 +++++++++++++++--- 5 files changed, 466 insertions(+), 52 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java index cc836cae945f9..e7c49c9d56c10 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java @@ -63,22 +63,23 @@ public void testBuildSliceReq() throws Exception { Assert.assertTrue(PipeTransferSliceReqBuilder.shouldSlice(req, bodySizeLimit)); Assert.assertEquals(3, PipeTransferSliceReqBuilder.getSliceCount(req, bodySizeLimit)); - final PipeTransferSliceReq firstSlice = - PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 0, 3, bodySizeLimit); - final PipeTransferSliceReq secondSlice = - PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 1, 3, bodySizeLimit); - final PipeTransferSliceReq thirdSlice = - PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 2, 3, bodySizeLimit); - - Assert.assertArrayEquals(new byte[] {0, 1, 2, 3}, firstSlice.getSliceBody()); - Assert.assertArrayEquals(new byte[] {4, 5, 6, 7}, secondSlice.getSliceBody()); - Assert.assertArrayEquals(new byte[] {8, 9}, thirdSlice.getSliceBody()); - Assert.assertEquals(0, firstSlice.getSliceIndex()); - Assert.assertEquals(1, secondSlice.getSliceIndex()); - Assert.assertEquals(2, thirdSlice.getSliceIndex()); - Assert.assertEquals(3, firstSlice.getSliceCount()); - Assert.assertEquals(req.getType(), firstSlice.getOriginReqType()); - Assert.assertEquals(10, firstSlice.getOriginBodySize()); + final byte[][] expectedBodies = { + new byte[] {0, 1, 2, 3}, new byte[] {4, 5, 6, 7}, new byte[] {8, 9} + }; + for (int i = 0; i < expectedBodies.length; i++) { + final PipeTransferSliceReq slice = + PipeTransferSliceReqBuilder.buildSliceReq(req, 123, i, 3, bodySizeLimit); + assertSlice(slice, 123, req.getType(), 10, expectedBodies[i], i, 3); + assertSlice( + PipeTransferSliceReq.fromTPipeTransferReq(copyOf(slice)), + 123, + req.getType(), + 10, + expectedBodies[i], + i, + 3); + } + Assert.assertEquals(0, req.body.position()); } @Test @@ -97,6 +98,34 @@ public void testShouldSliceOnlyForVersion1RequestsAboveThreshold() { createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 4), bodySizeLimit)); } + @Test + public void testGetSliceCountForBoundaryBodySizes() { + final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + + Assert.assertEquals( + 1, + PipeTransferSliceReqBuilder.getSliceCount( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), bodySizeLimit), + bodySizeLimit)); + Assert.assertEquals( + 2, + PipeTransferSliceReqBuilder.getSliceCount( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), bodySizeLimit + 1), + bodySizeLimit)); + Assert.assertEquals( + 2, + PipeTransferSliceReqBuilder.getSliceCount( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), bodySizeLimit * 2), + bodySizeLimit)); + } + + @Test + public void testSliceOrderIdIncreases() { + final int firstOrderId = PipeTransferSliceReqBuilder.nextSliceOrderId(); + + Assert.assertEquals(firstOrderId + 1, PipeTransferSliceReqBuilder.nextSliceOrderId()); + } + @Test public void testPipeTransferSliceReqFromLegacyV13Body() throws IOException { final TPipeTransferReq req = new TPipeTransferReq(); @@ -123,6 +152,32 @@ public void testPipeTransferSliceReqFromLegacyV13Body() throws IOException { Assert.assertEquals(2, sliceReq.getSliceCount()); } + private static void assertSlice( + final PipeTransferSliceReq sliceReq, + final int expectedOrderId, + final short expectedOriginReqType, + final int expectedOriginBodySize, + final byte[] expectedSliceBody, + final int expectedSliceIndex, + final int expectedSliceCount) { + Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), sliceReq.version); + Assert.assertEquals(PipeRequestType.TRANSFER_SLICE.getType(), sliceReq.type); + Assert.assertEquals(expectedOrderId, sliceReq.getOrderId()); + Assert.assertEquals(expectedOriginReqType, sliceReq.getOriginReqType()); + Assert.assertEquals(expectedOriginBodySize, sliceReq.getOriginBodySize()); + Assert.assertArrayEquals(expectedSliceBody, sliceReq.getSliceBody()); + Assert.assertEquals(expectedSliceIndex, sliceReq.getSliceIndex()); + Assert.assertEquals(expectedSliceCount, sliceReq.getSliceCount()); + } + + private static TPipeTransferReq copyOf(final TPipeTransferReq req) { + final TPipeTransferReq copy = new TPipeTransferReq(); + copy.version = req.version; + copy.type = req.type; + copy.body = req.body.duplicate(); + return copy; + } + private static TPipeTransferReq createReq(final byte version, final int bodySize) { final byte[] body = new byte[bodySize]; for (int i = 0; i < body.length; ++i) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java index ccdbc5086a5c9..44b602f8b7708 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java @@ -40,10 +40,7 @@ public class PipeTransferCompressedReqTest { @Test public void testPipeTransferCompressedReq() throws IOException { - final TPipeTransferReq originalReq = new TPipeTransferReq(); - originalReq.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); - originalReq.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType(); - originalReq.body = ByteBuffer.wrap(new byte[] {1, 2, 3, 4}); + final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); final TPipeTransferReq compressedReq = PipeTransferCompressedReq.toTPipeTransferReq( @@ -51,22 +48,57 @@ public void testPipeTransferCompressedReq() throws IOException { Collections.singletonList( PipeCompressorFactory.getCompressor( PipeCompressor.PipeCompressionType.GZIP.getIndex()))); - final TPipeTransferReq decompressedReq = - PipeTransferCompressedReq.fromTPipeTransferReq(compressedReq); Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), compressedReq.version); Assert.assertEquals(PipeRequestType.TRANSFER_COMPRESSED.getType(), compressedReq.type); - Assert.assertEquals(originalReq.version, decompressedReq.version); - Assert.assertEquals(originalReq.type, decompressedReq.type); - Assert.assertArrayEquals(originalReq.getBody(), decompressedReq.getBody()); + assertRoundTrip(originalReq, compressedReq); + } + + @Test + public void testPipeTransferCompressedReqWithMultipleCompressors() throws IOException { + final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); + + final TPipeTransferReq compressedReq = + PipeTransferCompressedReq.toTPipeTransferReq( + originalReq, + Arrays.asList( + PipeCompressorFactory.getCompressor( + PipeCompressor.PipeCompressionType.GZIP.getIndex()), + PipeCompressorFactory.getCompressor( + PipeCompressor.PipeCompressionType.LZ4.getIndex()))); + + assertRoundTrip(originalReq, compressedReq); + } + + @Test + public void testPipeTransferCompressedReqBytes() throws IOException { + final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); + final byte[] originalReqBytes = + BytesUtils.concatByteArrayList( + Arrays.asList( + new byte[] {originalReq.version}, + BytesUtils.shortToBytes(originalReq.type), + originalReq.getBody())); + final ByteBuffer compressedReqBuffer = + ByteBuffer.wrap( + PipeTransferCompressedReq.toTPipeTransferReqBytes( + originalReqBytes, + Collections.singletonList( + PipeCompressorFactory.getCompressor( + PipeCompressor.PipeCompressionType.GZIP.getIndex())))); + final TPipeTransferReq compressedReq = new TPipeTransferReq(); + compressedReq.version = ReadWriteIOUtils.readByte(compressedReqBuffer); + compressedReq.type = ReadWriteIOUtils.readShort(compressedReqBuffer); + compressedReq.body = compressedReqBuffer.slice(); + + Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), compressedReq.version); + Assert.assertEquals(PipeRequestType.TRANSFER_COMPRESSED.getType(), compressedReq.type); + assertRoundTrip(originalReq, compressedReq); } @Test public void testPipeTransferCompressedReqFromLegacyV13Body() throws IOException { - final TPipeTransferReq originalReq = new TPipeTransferReq(); - originalReq.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); - originalReq.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType(); - originalReq.body = ByteBuffer.wrap(new byte[] {1, 2, 3, 4}); + final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); final TPipeTransferReq compressedReq = new TPipeTransferReq(); compressedReq.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); @@ -78,6 +110,11 @@ public void testPipeTransferCompressedReqFromLegacyV13Body() throws IOException PipeCompressorFactory.getCompressor( PipeCompressor.PipeCompressionType.GZIP.getIndex()))); + assertRoundTrip(originalReq, compressedReq); + } + + private static void assertRoundTrip( + final TPipeTransferReq originalReq, final TPipeTransferReq compressedReq) throws IOException { final TPipeTransferReq decompressedReq = PipeTransferCompressedReq.fromTPipeTransferReq(compressedReq); @@ -86,6 +123,14 @@ public void testPipeTransferCompressedReqFromLegacyV13Body() throws IOException Assert.assertArrayEquals(originalReq.getBody(), decompressedReq.getBody()); } + private static TPipeTransferReq createReq(final byte[] body) { + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); + req.type = PipeRequestType.TRANSFER_TABLET_BINARY.getType(); + req.body = ByteBuffer.wrap(body); + return req; + } + private static ByteBuffer serializeLegacyCompressedBody( final TPipeTransferReq originalReq, final List compressors) throws IOException { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java new file mode 100644 index 0000000000000..a2a60ef4d6b79 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class PipeTransferFilePieceReqTest { + + private static final String FILE_NAME = "1-0-0-0.tsfile"; + private static final long START_WRITING_OFFSET = 12L; + private static final byte[] FILE_PIECE = new byte[] {1, 2, 3, 4}; + + @Test + public void testFilePieceReqRoundTripKeepsOffsetAndBody() throws IOException { + final DummyFilePieceReq req = + DummyFilePieceReq.toTPipeTransferReq(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE); + + Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); + Assert.assertEquals(PipeRequestType.TRANSFER_TS_FILE_PIECE.getType(), req.type); + Assert.assertEquals(FILE_NAME, req.getFileName()); + Assert.assertEquals(START_WRITING_OFFSET, req.getStartWritingOffset()); + Assert.assertArrayEquals(FILE_PIECE, req.getFilePiece()); + assertFilePieceBody(req.body.duplicate()); + + final DummyFilePieceReq deserializedReq = + (DummyFilePieceReq) new DummyFilePieceReq().translateFromTPipeTransferReq(copyOf(req)); + + Assert.assertEquals(req.version, deserializedReq.version); + Assert.assertEquals(req.type, deserializedReq.type); + Assert.assertEquals(FILE_NAME, deserializedReq.getFileName()); + Assert.assertEquals(START_WRITING_OFFSET, deserializedReq.getStartWritingOffset()); + Assert.assertArrayEquals(FILE_PIECE, deserializedReq.getFilePiece()); + } + + @Test + public void testFilePieceAirGapBytesKeepSameBodyFormat() throws IOException { + final ByteBuffer buffer = + ByteBuffer.wrap( + new DummyFilePieceReq() + .convertToTPipeTransferBytes(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE)); + + Assert.assertEquals( + IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); + Assert.assertEquals( + PipeRequestType.TRANSFER_TS_FILE_PIECE.getType(), ReadWriteIOUtils.readShort(buffer)); + assertFilePieceBody(buffer); + } + + private static void assertFilePieceBody(final ByteBuffer body) { + Assert.assertEquals(FILE_NAME, ReadWriteIOUtils.readString(body)); + Assert.assertEquals(START_WRITING_OFFSET, ReadWriteIOUtils.readLong(body)); + Assert.assertArrayEquals(FILE_PIECE, ReadWriteIOUtils.readBinary(body).getValues()); + Assert.assertFalse(body.hasRemaining()); + } + + private static TPipeTransferReq copyOf(final TPipeTransferReq req) { + final TPipeTransferReq copy = new TPipeTransferReq(); + copy.version = req.version; + copy.type = req.type; + copy.body = req.body.duplicate(); + return copy; + } + + private static class DummyFilePieceReq extends PipeTransferFilePieceReq { + + private static DummyFilePieceReq toTPipeTransferReq( + final String fileName, final long startWritingOffset, final byte[] filePiece) + throws IOException { + return (DummyFilePieceReq) + new DummyFilePieceReq() + .convertToTPipeTransferReq(fileName, startWritingOffset, filePiece); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_PIECE; + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java new file mode 100644 index 0000000000000..a07a71c8995fe --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; + +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class PipeTransferFileSealReqV1Test { + + private static final String FILE_NAME = "1-0-0-0.tsfile"; + private static final long FILE_LENGTH = 1024L; + + @Test + public void testFileSealReqV1RoundTripKeepsFileNameAndLength() throws IOException { + final DummyFileSealReqV1 req = DummyFileSealReqV1.toTPipeTransferReq(FILE_NAME, FILE_LENGTH); + + Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); + Assert.assertEquals(PipeRequestType.TRANSFER_TS_FILE_SEAL.getType(), req.type); + Assert.assertEquals(FILE_NAME, req.getFileName()); + Assert.assertEquals(FILE_LENGTH, req.getFileLength()); + assertFileSealBody(req.body.duplicate()); + + final DummyFileSealReqV1 deserializedReq = + (DummyFileSealReqV1) new DummyFileSealReqV1().translateFromTPipeTransferReq(copyOf(req)); + + Assert.assertEquals(req.version, deserializedReq.version); + Assert.assertEquals(req.type, deserializedReq.type); + Assert.assertEquals(FILE_NAME, deserializedReq.getFileName()); + Assert.assertEquals(FILE_LENGTH, deserializedReq.getFileLength()); + } + + @Test + public void testFileSealAirGapBytesKeepSameBodyFormat() throws IOException { + final ByteBuffer buffer = + ByteBuffer.wrap( + new DummyFileSealReqV1() + .convertToTPipeTransferSnapshotSealBytes(FILE_NAME, FILE_LENGTH)); + + Assert.assertEquals( + IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); + Assert.assertEquals( + PipeRequestType.TRANSFER_TS_FILE_SEAL.getType(), ReadWriteIOUtils.readShort(buffer)); + assertFileSealBody(buffer); + } + + private static void assertFileSealBody(final ByteBuffer body) { + Assert.assertEquals(FILE_NAME, ReadWriteIOUtils.readString(body)); + Assert.assertEquals(FILE_LENGTH, ReadWriteIOUtils.readLong(body)); + Assert.assertFalse(body.hasRemaining()); + } + + private static TPipeTransferReq copyOf(final TPipeTransferReq req) { + final TPipeTransferReq copy = new TPipeTransferReq(); + copy.version = req.version; + copy.type = req.type; + copy.body = req.body.duplicate(); + return copy; + } + + private static class DummyFileSealReqV1 extends PipeTransferFileSealReqV1 { + + private static DummyFileSealReqV1 toTPipeTransferReq( + final String fileName, final long fileLength) throws IOException { + return (DummyFileSealReqV1) + new DummyFileSealReqV1().convertToTPipeTransferReq(fileName, fileLength); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_SEAL; + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java index 4ac0e5cb7c333..f824b7cb31d28 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java @@ -19,47 +19,162 @@ package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; -import java.util.HashMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; public class PipeTransferFileSealReqV2Test { @Test - public void testLegacyV13SnapshotSealCapturesTreeOnly() { - final Map parameters = new HashMap<>(); - - Assert.assertTrue(PipeTransferFileSealReqV2.isTreeModelDataAllowedToBeCaptured(parameters)); - Assert.assertFalse(PipeTransferFileSealReqV2.isTableModelDataAllowedToBeCaptured(parameters)); + public void testSnapshotSealModelCaptureRules() { + assertCapture("legacy v1.3", Collections.emptyMap(), true, false); + assertCapture( + "non-model parameter only", + markerParameters(PipeTransferFileSealReqV2.DATABASE_PATTERN), + true, + false); + assertCapture("tree only", markerParameters(PipeTransferFileSealReqV2.TREE), true, false); + assertCapture("table only", markerParameters(PipeTransferFileSealReqV2.TABLE), false, true); + assertCapture( + "tree and table", + markerParameters(PipeTransferFileSealReqV2.TREE, PipeTransferFileSealReqV2.TABLE), + true, + true); } @Test - public void testExplicitTreeOnlySnapshotSealCapturesTreeOnly() { - final Map parameters = new HashMap<>(); - parameters.put(PipeTransferFileSealReqV2.TREE, ""); + public void testSnapshotSealReqV2RoundTripKeepsFilesAndParameters() throws IOException { + final List fileNames = Arrays.asList("schema.snapshot", "template.snapshot"); + final List fileLengths = Arrays.asList(12L, 34L); + final Map parameters = snapshotParameters(); + + final DummyFileSealReqV2 req = + DummyFileSealReqV2.toTPipeTransferReq(fileNames, fileLengths, parameters); + + Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); + Assert.assertEquals(PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), req.type); + Assert.assertEquals(fileNames, req.getFileNames()); + Assert.assertEquals(fileLengths, req.getFileLengths()); + Assert.assertEquals(parameters, req.getParameters()); + assertSnapshotSealBody(req.body.duplicate(), fileNames, fileLengths, parameters); + + final DummyFileSealReqV2 deserializedReq = + (DummyFileSealReqV2) new DummyFileSealReqV2().translateFromTPipeTransferReq(copyOf(req)); - Assert.assertTrue(PipeTransferFileSealReqV2.isTreeModelDataAllowedToBeCaptured(parameters)); - Assert.assertFalse(PipeTransferFileSealReqV2.isTableModelDataAllowedToBeCaptured(parameters)); + Assert.assertEquals(req.version, deserializedReq.version); + Assert.assertEquals(req.type, deserializedReq.type); + Assert.assertEquals(fileNames, deserializedReq.getFileNames()); + Assert.assertEquals(fileLengths, deserializedReq.getFileLengths()); + Assert.assertEquals(parameters, deserializedReq.getParameters()); } @Test - public void testExplicitTableOnlySnapshotSealCapturesTableOnly() { - final Map parameters = new HashMap<>(); - parameters.put(PipeTransferFileSealReqV2.TABLE, ""); + public void testSnapshotSealAirGapBytesKeepSameBodyFormat() throws IOException { + final List fileNames = Arrays.asList("schema.snapshot", "template.snapshot"); + final List fileLengths = Arrays.asList(12L, 34L); + final Map parameters = snapshotParameters(); - Assert.assertFalse(PipeTransferFileSealReqV2.isTreeModelDataAllowedToBeCaptured(parameters)); - Assert.assertTrue(PipeTransferFileSealReqV2.isTableModelDataAllowedToBeCaptured(parameters)); + final ByteBuffer buffer = + ByteBuffer.wrap( + new DummyFileSealReqV2() + .convertToTPipeTransferSnapshotSealBytes(fileNames, fileLengths, parameters)); + + Assert.assertEquals( + IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); + Assert.assertEquals( + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), + ReadWriteIOUtils.readShort(buffer)); + assertSnapshotSealBody(buffer, fileNames, fileLengths, parameters); } - @Test - public void testExplicitTreeAndTableSnapshotSealCapturesBoth() { - final Map parameters = new HashMap<>(); - parameters.put(PipeTransferFileSealReqV2.TREE, ""); - parameters.put(PipeTransferFileSealReqV2.TABLE, ""); + private static void assertCapture( + final String caseName, + final Map parameters, + final boolean expectedTreeCaptured, + final boolean expectedTableCaptured) { + Assert.assertEquals( + caseName, + expectedTreeCaptured, + PipeTransferFileSealReqV2.isTreeModelDataAllowedToBeCaptured(parameters)); + Assert.assertEquals( + caseName, + expectedTableCaptured, + PipeTransferFileSealReqV2.isTableModelDataAllowedToBeCaptured(parameters)); + } + + private static void assertSnapshotSealBody( + final ByteBuffer body, + final List expectedFileNames, + final List expectedFileLengths, + final Map expectedParameters) { + final int fileNameSize = ReadWriteIOUtils.readInt(body); + Assert.assertEquals(expectedFileNames.size(), fileNameSize); + for (final String expectedFileName : expectedFileNames) { + Assert.assertEquals(expectedFileName, ReadWriteIOUtils.readString(body)); + } + + final int fileLengthSize = ReadWriteIOUtils.readInt(body); + Assert.assertEquals(expectedFileLengths.size(), fileLengthSize); + for (final Long expectedFileLength : expectedFileLengths) { + Assert.assertEquals(expectedFileLength.longValue(), ReadWriteIOUtils.readLong(body)); + } + + final int parameterSize = ReadWriteIOUtils.readInt(body); + final Map parameters = new LinkedHashMap<>(); + for (int i = 0; i < parameterSize; i++) { + parameters.put(ReadWriteIOUtils.readString(body), ReadWriteIOUtils.readString(body)); + } + Assert.assertEquals(expectedParameters, parameters); + Assert.assertFalse(body.hasRemaining()); + } + + private static Map markerParameters(final String... keys) { + final Map parameters = new LinkedHashMap<>(); + for (final String key : keys) { + parameters.put(key, ""); + } + return parameters; + } + + private static Map snapshotParameters() { + final Map parameters = + markerParameters(PipeTransferFileSealReqV2.TREE, PipeTransferFileSealReqV2.TABLE); + parameters.put(PipeTransferFileSealReqV2.DATABASE_PATTERN, "root.sg.*"); + return parameters; + } + + private static TPipeTransferReq copyOf(final TPipeTransferReq req) { + final TPipeTransferReq copy = new TPipeTransferReq(); + copy.version = req.version; + copy.type = req.type; + copy.body = req.body.duplicate(); + return copy; + } + + private static class DummyFileSealReqV2 extends PipeTransferFileSealReqV2 { + + private static DummyFileSealReqV2 toTPipeTransferReq( + final List fileNames, + final List fileLengths, + final Map parameters) + throws IOException { + return (DummyFileSealReqV2) + new DummyFileSealReqV2().convertToTPipeTransferReq(fileNames, fileLengths, parameters); + } - Assert.assertTrue(PipeTransferFileSealReqV2.isTreeModelDataAllowedToBeCaptured(parameters)); - Assert.assertTrue(PipeTransferFileSealReqV2.isTableModelDataAllowedToBeCaptured(parameters)); + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL; + } } } From 15cf51664be17ab1f5e520205c1b5a6224a18a1d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:49:47 +0800 Subject: [PATCH 3/9] Improve pipe transfer request tests --- .../thrift/PipeTransferReqTestUtils.java | 68 +++++++++++++++++++ .../PipeTransferSliceReqBuilderTest.java | 15 ++-- .../PipeTransferCompressedReqTest.java | 67 +++++++++++++----- .../request/PipeTransferFilePieceReqTest.java | 17 ++--- .../PipeTransferFileSealReqV1Test.java | 17 ++--- .../PipeTransferFileSealReqV2Test.java | 17 ++--- 6 files changed, 138 insertions(+), 63 deletions(-) create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java new file mode 100644 index 0000000000000..81b4b82fd493b --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.payload.thrift; + +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +public final class PipeTransferReqTestUtils { + + public static void assertVersionAndType( + final TPipeTransferReq req, + final IoTDBSinkRequestVersion expectedVersion, + final PipeRequestType expectedType) { + Assert.assertEquals(expectedVersion.getVersion(), req.version); + Assert.assertEquals(expectedType.getType(), req.type); + } + + public static TPipeTransferReq copyOf(final TPipeTransferReq req) { + final TPipeTransferReq copy = new TPipeTransferReq(); + copy.version = req.version; + copy.type = req.type; + copy.body = req.body.duplicate(); + return copy; + } + + public static byte[] toTransferReqBytes(final TPipeTransferReq req) { + return BytesUtils.concatByteArrayList( + Arrays.asList(new byte[] {req.version}, BytesUtils.shortToBytes(req.type), req.getBody())); + } + + public static TPipeTransferReq readTransferReqFrom(final byte[] reqBytes) { + final ByteBuffer reqBuffer = ByteBuffer.wrap(reqBytes); + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = ReadWriteIOUtils.readByte(reqBuffer); + req.type = ReadWriteIOUtils.readShort(reqBuffer); + req.body = reqBuffer.slice(); + return req; + } + + private PipeTransferReqTestUtils() { + // Utility class + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java index e7c49c9d56c10..4857328ca13c4 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java @@ -38,6 +38,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; + public class PipeTransferSliceReqBuilderTest { private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); @@ -160,8 +163,8 @@ private static void assertSlice( final byte[] expectedSliceBody, final int expectedSliceIndex, final int expectedSliceCount) { - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), sliceReq.version); - Assert.assertEquals(PipeRequestType.TRANSFER_SLICE.getType(), sliceReq.type); + assertVersionAndType( + sliceReq, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_SLICE); Assert.assertEquals(expectedOrderId, sliceReq.getOrderId()); Assert.assertEquals(expectedOriginReqType, sliceReq.getOriginReqType()); Assert.assertEquals(expectedOriginBodySize, sliceReq.getOriginBodySize()); @@ -170,14 +173,6 @@ private static void assertSlice( Assert.assertEquals(expectedSliceCount, sliceReq.getSliceCount()); } - private static TPipeTransferReq copyOf(final TPipeTransferReq req) { - final TPipeTransferReq copy = new TPipeTransferReq(); - copy.version = req.version; - copy.type = req.type; - copy.body = req.body.duplicate(); - return copy; - } - private static TPipeTransferReq createReq(final byte version, final int bodySize) { final byte[] body = new byte[bodySize]; for (int i = 0; i < body.length; ++i) { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java index 44b602f8b7708..6ca8fa0f9cdc4 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.pipe.sink.compressor.PipeCompressor; import org.apache.iotdb.commons.pipe.sink.compressor.PipeCompressorFactory; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -36,6 +38,10 @@ import java.util.Collections; import java.util.List; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.readTransferReqFrom; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.toTransferReqBytes; + public class PipeTransferCompressedReqTest { @Test @@ -49,8 +55,8 @@ public void testPipeTransferCompressedReq() throws IOException { PipeCompressorFactory.getCompressor( PipeCompressor.PipeCompressionType.GZIP.getIndex()))); - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), compressedReq.version); - Assert.assertEquals(PipeRequestType.TRANSFER_COMPRESSED.getType(), compressedReq.type); + assertVersionAndType( + compressedReq, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_COMPRESSED); assertRoundTrip(originalReq, compressedReq); } @@ -73,26 +79,16 @@ public void testPipeTransferCompressedReqWithMultipleCompressors() throws IOExce @Test public void testPipeTransferCompressedReqBytes() throws IOException { final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); - final byte[] originalReqBytes = - BytesUtils.concatByteArrayList( - Arrays.asList( - new byte[] {originalReq.version}, - BytesUtils.shortToBytes(originalReq.type), - originalReq.getBody())); - final ByteBuffer compressedReqBuffer = - ByteBuffer.wrap( + final TPipeTransferReq compressedReq = + readTransferReqFrom( PipeTransferCompressedReq.toTPipeTransferReqBytes( - originalReqBytes, + toTransferReqBytes(originalReq), Collections.singletonList( PipeCompressorFactory.getCompressor( PipeCompressor.PipeCompressionType.GZIP.getIndex())))); - final TPipeTransferReq compressedReq = new TPipeTransferReq(); - compressedReq.version = ReadWriteIOUtils.readByte(compressedReqBuffer); - compressedReq.type = ReadWriteIOUtils.readShort(compressedReqBuffer); - compressedReq.body = compressedReqBuffer.slice(); - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), compressedReq.version); - Assert.assertEquals(PipeRequestType.TRANSFER_COMPRESSED.getType(), compressedReq.type); + assertVersionAndType( + compressedReq, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_COMPRESSED); assertRoundTrip(originalReq, compressedReq); } @@ -113,6 +109,28 @@ public void testPipeTransferCompressedReqFromLegacyV13Body() throws IOException assertRoundTrip(originalReq, compressedReq); } + @Test + public void testPipeTransferCompressedReqRejectsInvalidUncompressedLength() { + Assert.assertThrows( + IllegalArgumentException.class, + () -> + PipeTransferCompressedReq.fromTPipeTransferReq( + createCompressedReqWithUncompressedLength(-1))); + + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final int originalMaxLength = commonConfig.getPipeReceiverReqDecompressedMaxLengthInBytes(); + try { + commonConfig.setPipeReceiverReqDecompressedMaxLengthInBytes(8); + Assert.assertThrows( + IllegalArgumentException.class, + () -> + PipeTransferCompressedReq.fromTPipeTransferReq( + createCompressedReqWithUncompressedLength(9))); + } finally { + commonConfig.setPipeReceiverReqDecompressedMaxLengthInBytes(originalMaxLength); + } + } + private static void assertRoundTrip( final TPipeTransferReq originalReq, final TPipeTransferReq compressedReq) throws IOException { final TPipeTransferReq decompressedReq = @@ -131,6 +149,21 @@ private static TPipeTransferReq createReq(final byte[] body) { return req; } + private static TPipeTransferReq createCompressedReqWithUncompressedLength( + final int uncompressedLength) throws IOException { + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); + req.type = PipeRequestType.TRANSFER_COMPRESSED.getType(); + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write((byte) 1, outputStream); + ReadWriteIOUtils.write(PipeCompressor.PipeCompressionType.GZIP.getIndex(), outputStream); + ReadWriteIOUtils.write(uncompressedLength, outputStream); + req.body = ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + return req; + } + private static ByteBuffer serializeLegacyCompressedBody( final TPipeTransferReq originalReq, final List compressors) throws IOException { diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java index a2a60ef4d6b79..630baec704c27 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java @@ -19,8 +19,6 @@ package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; -import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; - import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; @@ -28,6 +26,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; + public class PipeTransferFilePieceReqTest { private static final String FILE_NAME = "1-0-0-0.tsfile"; @@ -39,8 +40,8 @@ public void testFilePieceReqRoundTripKeepsOffsetAndBody() throws IOException { final DummyFilePieceReq req = DummyFilePieceReq.toTPipeTransferReq(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE); - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); - Assert.assertEquals(PipeRequestType.TRANSFER_TS_FILE_PIECE.getType(), req.type); + assertVersionAndType( + req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_TS_FILE_PIECE); Assert.assertEquals(FILE_NAME, req.getFileName()); Assert.assertEquals(START_WRITING_OFFSET, req.getStartWritingOffset()); Assert.assertArrayEquals(FILE_PIECE, req.getFilePiece()); @@ -77,14 +78,6 @@ private static void assertFilePieceBody(final ByteBuffer body) { Assert.assertFalse(body.hasRemaining()); } - private static TPipeTransferReq copyOf(final TPipeTransferReq req) { - final TPipeTransferReq copy = new TPipeTransferReq(); - copy.version = req.version; - copy.type = req.type; - copy.body = req.body.duplicate(); - return copy; - } - private static class DummyFilePieceReq extends PipeTransferFilePieceReq { private static DummyFilePieceReq toTPipeTransferReq( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java index a07a71c8995fe..acfe0f6ef0336 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java @@ -19,8 +19,6 @@ package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; -import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; - import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; @@ -28,6 +26,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; + public class PipeTransferFileSealReqV1Test { private static final String FILE_NAME = "1-0-0-0.tsfile"; @@ -37,8 +38,8 @@ public class PipeTransferFileSealReqV1Test { public void testFileSealReqV1RoundTripKeepsFileNameAndLength() throws IOException { final DummyFileSealReqV1 req = DummyFileSealReqV1.toTPipeTransferReq(FILE_NAME, FILE_LENGTH); - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); - Assert.assertEquals(PipeRequestType.TRANSFER_TS_FILE_SEAL.getType(), req.type); + assertVersionAndType( + req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_TS_FILE_SEAL); Assert.assertEquals(FILE_NAME, req.getFileName()); Assert.assertEquals(FILE_LENGTH, req.getFileLength()); assertFileSealBody(req.body.duplicate()); @@ -72,14 +73,6 @@ private static void assertFileSealBody(final ByteBuffer body) { Assert.assertFalse(body.hasRemaining()); } - private static TPipeTransferReq copyOf(final TPipeTransferReq req) { - final TPipeTransferReq copy = new TPipeTransferReq(); - copy.version = req.version; - copy.type = req.type; - copy.body = req.body.duplicate(); - return copy; - } - private static class DummyFileSealReqV1 extends PipeTransferFileSealReqV1 { private static DummyFileSealReqV1 toTPipeTransferReq( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java index f824b7cb31d28..85e991e1053c7 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java @@ -19,8 +19,6 @@ package org.apache.iotdb.commons.pipe.sink.payload.thrift.request; -import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; - import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; @@ -33,6 +31,9 @@ import java.util.List; import java.util.Map; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; + public class PipeTransferFileSealReqV2Test { @Test @@ -61,8 +62,8 @@ public void testSnapshotSealReqV2RoundTripKeepsFilesAndParameters() throws IOExc final DummyFileSealReqV2 req = DummyFileSealReqV2.toTPipeTransferReq(fileNames, fileLengths, parameters); - Assert.assertEquals(IoTDBSinkRequestVersion.VERSION_1.getVersion(), req.version); - Assert.assertEquals(PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), req.type); + assertVersionAndType( + req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL); Assert.assertEquals(fileNames, req.getFileNames()); Assert.assertEquals(fileLengths, req.getFileLengths()); Assert.assertEquals(parameters, req.getParameters()); @@ -153,14 +154,6 @@ private static Map snapshotParameters() { return parameters; } - private static TPipeTransferReq copyOf(final TPipeTransferReq req) { - final TPipeTransferReq copy = new TPipeTransferReq(); - copy.version = req.version; - copy.type = req.type; - copy.body = req.body.duplicate(); - return copy; - } - private static class DummyFileSealReqV2 extends PipeTransferFileSealReqV2 { private static DummyFileSealReqV2 toTPipeTransferReq( From b749d8c85014692cb44dea22e6e7b00c40284714 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:12:53 +0800 Subject: [PATCH 4/9] Improve pipe transfer request tests --- .../pipe/receiver/IoTDBFileReceiverTest.java | 160 ++++++++++-------- .../thrift/PipeTransferReqTestUtils.java | 13 ++ .../PipeTransferSliceReqBuilderTest.java | 7 + .../PipeTransferCompressedReqTest.java | 12 ++ .../request/PipeTransferFilePieceReqTest.java | 68 +++++--- .../PipeTransferFileSealReqV1Test.java | 54 +++--- .../PipeTransferFileSealReqV2Test.java | 78 +++++---- 7 files changed, 247 insertions(+), 145 deletions(-) diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java index 5a1ed3e653aec..7540acf21a7e2 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java @@ -47,101 +47,124 @@ public class IoTDBFileReceiverTest { @Test public void testRejectPathTraversalFileName() throws Exception { - final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); - final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); - try { - final IOException exception = - Assert.assertThrows( - IOException.class, () -> receiver.createWritingFile("../outside.tsfile", true)); - Assert.assertTrue(exception.getMessage().contains("Illegal fileName")); - } finally { - receiver.handleExit(); - } + withReceiver( + receiver -> { + final IOException exception = + Assert.assertThrows( + IOException.class, () -> receiver.createWritingFile("../outside.tsfile", true)); + Assert.assertTrue(exception.getMessage().contains("Illegal fileName")); + }); } @Test public void testAllowNormalFileName() throws Exception { - final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); - final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); - try { - receiver.createWritingFile("normal.tsfile", true); - Assert.assertTrue(receiver.getWritingFileInBaseDir("normal.tsfile").exists()); - } finally { - receiver.handleExit(); - } + withReceiver( + receiver -> { + receiver.createWritingFile("normal.tsfile", true); + Assert.assertTrue(receiver.getWritingFileInBaseDir("normal.tsfile").exists()); + }); } @Test public void testRejectPathTraversalFileNameInSealRequest() throws Exception { - final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); - final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); - try { - receiver.createWritingFile("normal.tsfile", false); - - final TPipeTransferResp response = - receiver.sealFiles( - Arrays.asList("../outside.mod", "normal.tsfile"), Arrays.asList(0L, 0L)); - - Assert.assertEquals( - TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), response.getStatus().getCode()); - Assert.assertTrue(response.getStatus().getMessage().contains("Illegal fileName")); - } finally { - receiver.handleExit(); - } + withReceiver( + receiver -> { + receiver.createWritingFile("normal.tsfile", false); + + final TPipeTransferResp response = + receiver.sealFiles( + Arrays.asList("../outside.mod", "normal.tsfile"), Arrays.asList(0L, 0L)); + + Assert.assertEquals( + TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), + response.getStatus().getCode()); + Assert.assertTrue(response.getStatus().getMessage().contains("Illegal fileName")); + }); } @Test public void testHandshakeResetsWritingFileState() throws Exception { - final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); - final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); - try { - receiver.handshake(); - receiver.createWritingFile("normal.tsfile", true); - receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); - - final File oldReceiverDir = receiver.getCurrentReceiverDir(); - Assert.assertNotNull(receiver.getCurrentWritingFile()); - Assert.assertNotNull(receiver.getCurrentWritingFileWriter()); - - receiver.handshake(); + withReceiver( + receiver -> { + receiver.handshake(); + receiver.createWritingFile("normal.tsfile", true); + receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); + + final File oldReceiverDir = receiver.getCurrentReceiverDir(); + Assert.assertNotNull(receiver.getCurrentWritingFile()); + Assert.assertNotNull(receiver.getCurrentWritingFileWriter()); + + receiver.handshake(); + + Assert.assertFalse(oldReceiverDir.exists()); + Assert.assertNull(receiver.getCurrentWritingFile()); + Assert.assertNull(receiver.getCurrentWritingFileWriter()); + Assert.assertNotEquals( + oldReceiverDir.getAbsolutePath(), receiver.getCurrentReceiverDir().getAbsolutePath()); + }); + } - Assert.assertFalse(oldReceiverDir.exists()); - Assert.assertNull(receiver.getCurrentWritingFile()); - Assert.assertNull(receiver.getCurrentWritingFileWriter()); - Assert.assertNotEquals( - oldReceiverDir.getAbsolutePath(), receiver.getCurrentReceiverDir().getAbsolutePath()); - } finally { - receiver.handleExit(); - } + @Test + public void testSealFileV1SuccessKeepsTransferredFileForLoader() throws Exception { + withReceiver( + receiver -> { + receiver.createWritingFile("normal.tsfile", true); + receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); + + final File transferredFile = receiver.getWritingFileInBaseDir("normal.tsfile"); + final TPipeTransferResp response = receiver.sealFileV1("normal.tsfile", 3L); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + Assert.assertTrue(transferredFile.exists()); + Assert.assertEquals(transferredFile.getAbsolutePath(), receiver.getLoadedFileV1Path()); + Assert.assertNull(receiver.getCurrentWritingFile()); + Assert.assertNull(receiver.getCurrentWritingFileWriter()); + }); } @Test public void testSealFileV1FailureDeletesTransferredFile() throws Exception { + withReceiver( + receiver -> { + receiver.createWritingFile("normal.tsfile", true); + receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); + receiver.setLoadFileV1Status( + new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode())); + + final File transferredFile = receiver.getWritingFileInBaseDir("normal.tsfile"); + final TPipeTransferResp response = receiver.sealFileV1("normal.tsfile", 3L); + + Assert.assertEquals( + TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), + response.getStatus().getCode()); + Assert.assertFalse(transferredFile.exists()); + Assert.assertNull(receiver.getCurrentWritingFile()); + Assert.assertNull(receiver.getCurrentWritingFileWriter()); + }); + } + + private static void withReceiver(final ReceiverConsumer action) throws Exception { final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); try { - receiver.createWritingFile("normal.tsfile", true); - receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); - receiver.setLoadFileV1Status( - new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode())); - - final File transferredFile = receiver.getWritingFileInBaseDir("normal.tsfile"); - final TPipeTransferResp response = receiver.sealFileV1("normal.tsfile", 3L); - - Assert.assertEquals( - TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), response.getStatus().getCode()); - Assert.assertFalse(transferredFile.exists()); - Assert.assertNull(receiver.getCurrentWritingFile()); - Assert.assertNull(receiver.getCurrentWritingFileWriter()); + action.accept(receiver); } finally { receiver.handleExit(); } } + @FunctionalInterface + private interface ReceiverConsumer { + + void accept(DummyFileReceiver receiver) throws Exception; + } + private static class DummyFileReceiver extends IoTDBFileReceiver { private final File receiverFileBaseDir; private TSStatus loadFileV1Status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + private String loadedFileV1Path; DummyFileReceiver(final File baseDir) { receiverFileBaseDir = baseDir; @@ -166,6 +189,10 @@ void setLoadFileV1Status(final TSStatus status) { loadFileV1Status = status; } + String getLoadedFileV1Path() { + return loadedFileV1Path; + } + TPipeTransferResp sealFileV1(final String fileName, final long fileLength) throws IOException { return handleTransferFileSealV1(DummyFileSealReqV1.toTPipeTransferReq(fileName, fileLength)); } @@ -231,6 +258,7 @@ protected TSStatus login() { @Override protected TSStatus loadFileV1( final PipeTransferFileSealReqV1 req, final String fileAbsolutePath) { + loadedFileV1Path = fileAbsolutePath; return loadFileV1Status; } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java index 81b4b82fd493b..f1ebca78dd361 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/PipeTransferReqTestUtils.java @@ -29,6 +29,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.function.Consumer; public final class PipeTransferReqTestUtils { @@ -40,6 +41,18 @@ public static void assertVersionAndType( Assert.assertEquals(expectedType.getType(), req.type); } + public static void assertAirGapReqBytes( + final byte[] reqBytes, + final IoTDBSinkRequestVersion expectedVersion, + final PipeRequestType expectedType, + final Consumer bodyAssertion) { + final ByteBuffer reqBuffer = ByteBuffer.wrap(reqBytes); + Assert.assertEquals(expectedVersion.getVersion(), ReadWriteIOUtils.readByte(reqBuffer)); + Assert.assertEquals(expectedType.getType(), ReadWriteIOUtils.readShort(reqBuffer)); + bodyAssertion.accept(reqBuffer); + Assert.assertFalse(reqBuffer.hasRemaining()); + } + public static TPipeTransferReq copyOf(final TPipeTransferReq req) { final TPipeTransferReq copy = new TPipeTransferReq(); copy.version = req.version; diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java index 4857328ca13c4..56152ccdd2470 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java @@ -89,6 +89,9 @@ public void testBuildSliceReq() throws Exception { public void testShouldSliceOnlyForVersion1RequestsAboveThreshold() { final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + Assert.assertFalse( + PipeTransferSliceReqBuilder.shouldSlice( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 0), bodySizeLimit)); Assert.assertFalse( PipeTransferSliceReqBuilder.shouldSlice( createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 3), bodySizeLimit)); @@ -105,6 +108,10 @@ public void testShouldSliceOnlyForVersion1RequestsAboveThreshold() { public void testGetSliceCountForBoundaryBodySizes() { final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + Assert.assertEquals( + 0, + PipeTransferSliceReqBuilder.getSliceCount( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 0), bodySizeLimit)); Assert.assertEquals( 1, PipeTransferSliceReqBuilder.getSliceCount( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java index 6ca8fa0f9cdc4..29e25c6d3129a 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferCompressedReqTest.java @@ -76,6 +76,18 @@ public void testPipeTransferCompressedReqWithMultipleCompressors() throws IOExce assertRoundTrip(originalReq, compressedReq); } + @Test + public void testPipeTransferCompressedReqWithNoCompressor() throws IOException { + final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); + + final TPipeTransferReq compressedReq = + PipeTransferCompressedReq.toTPipeTransferReq(originalReq, Collections.emptyList()); + + assertVersionAndType( + compressedReq, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_COMPRESSED); + assertRoundTrip(originalReq, compressedReq); + } + @Test public void testPipeTransferCompressedReqBytes() throws IOException { final TPipeTransferReq originalReq = createReq(new byte[] {1, 2, 3, 4}); diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java index 630baec704c27..89b882f500a30 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFilePieceReqTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertAirGapReqBytes; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; @@ -37,44 +38,59 @@ public class PipeTransferFilePieceReqTest { @Test public void testFilePieceReqRoundTripKeepsOffsetAndBody() throws IOException { + assertFilePieceReqRoundTrip(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE); + } + + @Test + public void testEmptyFilePieceReqRoundTripKeepsOffsetAndBody() throws IOException { + assertFilePieceReqRoundTrip(FILE_NAME, 0L, new byte[0]); + } + + @Test + public void testFilePieceAirGapBytesKeepSameBodyFormat() throws IOException { + assertAirGapReqBytes( + new DummyFilePieceReq() + .convertToTPipeTransferBytes(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE), + IoTDBSinkRequestVersion.VERSION_1, + PipeRequestType.TRANSFER_TS_FILE_PIECE, + body -> assertFilePieceBody(body, FILE_NAME, START_WRITING_OFFSET, FILE_PIECE)); + } + + private static void assertFilePieceReqRoundTrip( + final String expectedFileName, + final long expectedStartWritingOffset, + final byte[] expectedFilePiece) + throws IOException { final DummyFilePieceReq req = - DummyFilePieceReq.toTPipeTransferReq(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE); + DummyFilePieceReq.toTPipeTransferReq( + expectedFileName, expectedStartWritingOffset, expectedFilePiece); assertVersionAndType( req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_TS_FILE_PIECE); - Assert.assertEquals(FILE_NAME, req.getFileName()); - Assert.assertEquals(START_WRITING_OFFSET, req.getStartWritingOffset()); - Assert.assertArrayEquals(FILE_PIECE, req.getFilePiece()); - assertFilePieceBody(req.body.duplicate()); + Assert.assertEquals(expectedFileName, req.getFileName()); + Assert.assertEquals(expectedStartWritingOffset, req.getStartWritingOffset()); + Assert.assertArrayEquals(expectedFilePiece, req.getFilePiece()); + assertFilePieceBody( + req.body.duplicate(), expectedFileName, expectedStartWritingOffset, expectedFilePiece); final DummyFilePieceReq deserializedReq = (DummyFilePieceReq) new DummyFilePieceReq().translateFromTPipeTransferReq(copyOf(req)); Assert.assertEquals(req.version, deserializedReq.version); Assert.assertEquals(req.type, deserializedReq.type); - Assert.assertEquals(FILE_NAME, deserializedReq.getFileName()); - Assert.assertEquals(START_WRITING_OFFSET, deserializedReq.getStartWritingOffset()); - Assert.assertArrayEquals(FILE_PIECE, deserializedReq.getFilePiece()); - } - - @Test - public void testFilePieceAirGapBytesKeepSameBodyFormat() throws IOException { - final ByteBuffer buffer = - ByteBuffer.wrap( - new DummyFilePieceReq() - .convertToTPipeTransferBytes(FILE_NAME, START_WRITING_OFFSET, FILE_PIECE)); - - Assert.assertEquals( - IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); - Assert.assertEquals( - PipeRequestType.TRANSFER_TS_FILE_PIECE.getType(), ReadWriteIOUtils.readShort(buffer)); - assertFilePieceBody(buffer); + Assert.assertEquals(expectedFileName, deserializedReq.getFileName()); + Assert.assertEquals(expectedStartWritingOffset, deserializedReq.getStartWritingOffset()); + Assert.assertArrayEquals(expectedFilePiece, deserializedReq.getFilePiece()); } - private static void assertFilePieceBody(final ByteBuffer body) { - Assert.assertEquals(FILE_NAME, ReadWriteIOUtils.readString(body)); - Assert.assertEquals(START_WRITING_OFFSET, ReadWriteIOUtils.readLong(body)); - Assert.assertArrayEquals(FILE_PIECE, ReadWriteIOUtils.readBinary(body).getValues()); + private static void assertFilePieceBody( + final ByteBuffer body, + final String expectedFileName, + final long expectedStartWritingOffset, + final byte[] expectedFilePiece) { + Assert.assertEquals(expectedFileName, ReadWriteIOUtils.readString(body)); + Assert.assertEquals(expectedStartWritingOffset, ReadWriteIOUtils.readLong(body)); + Assert.assertArrayEquals(expectedFilePiece, ReadWriteIOUtils.readBinary(body).getValues()); Assert.assertFalse(body.hasRemaining()); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java index acfe0f6ef0336..f8f0c943497c2 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV1Test.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertAirGapReqBytes; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; @@ -36,40 +37,47 @@ public class PipeTransferFileSealReqV1Test { @Test public void testFileSealReqV1RoundTripKeepsFileNameAndLength() throws IOException { - final DummyFileSealReqV1 req = DummyFileSealReqV1.toTPipeTransferReq(FILE_NAME, FILE_LENGTH); + assertFileSealReqRoundTrip(FILE_NAME, FILE_LENGTH); + } + + @Test + public void testEmptyFileSealReqV1RoundTripKeepsFileNameAndLength() throws IOException { + assertFileSealReqRoundTrip(FILE_NAME, 0L); + } + + @Test + public void testFileSealAirGapBytesKeepSameBodyFormat() throws IOException { + assertAirGapReqBytes( + new DummyFileSealReqV1().convertToTPipeTransferSnapshotSealBytes(FILE_NAME, FILE_LENGTH), + IoTDBSinkRequestVersion.VERSION_1, + PipeRequestType.TRANSFER_TS_FILE_SEAL, + body -> assertFileSealBody(body, FILE_NAME, FILE_LENGTH)); + } + + private static void assertFileSealReqRoundTrip( + final String expectedFileName, final long expectedFileLength) throws IOException { + final DummyFileSealReqV1 req = + DummyFileSealReqV1.toTPipeTransferReq(expectedFileName, expectedFileLength); assertVersionAndType( req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_TS_FILE_SEAL); - Assert.assertEquals(FILE_NAME, req.getFileName()); - Assert.assertEquals(FILE_LENGTH, req.getFileLength()); - assertFileSealBody(req.body.duplicate()); + Assert.assertEquals(expectedFileName, req.getFileName()); + Assert.assertEquals(expectedFileLength, req.getFileLength()); + assertFileSealBody(req.body.duplicate(), expectedFileName, expectedFileLength); final DummyFileSealReqV1 deserializedReq = (DummyFileSealReqV1) new DummyFileSealReqV1().translateFromTPipeTransferReq(copyOf(req)); Assert.assertEquals(req.version, deserializedReq.version); Assert.assertEquals(req.type, deserializedReq.type); - Assert.assertEquals(FILE_NAME, deserializedReq.getFileName()); - Assert.assertEquals(FILE_LENGTH, deserializedReq.getFileLength()); - } - - @Test - public void testFileSealAirGapBytesKeepSameBodyFormat() throws IOException { - final ByteBuffer buffer = - ByteBuffer.wrap( - new DummyFileSealReqV1() - .convertToTPipeTransferSnapshotSealBytes(FILE_NAME, FILE_LENGTH)); - - Assert.assertEquals( - IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); - Assert.assertEquals( - PipeRequestType.TRANSFER_TS_FILE_SEAL.getType(), ReadWriteIOUtils.readShort(buffer)); - assertFileSealBody(buffer); + Assert.assertEquals(expectedFileName, deserializedReq.getFileName()); + Assert.assertEquals(expectedFileLength, deserializedReq.getFileLength()); } - private static void assertFileSealBody(final ByteBuffer body) { - Assert.assertEquals(FILE_NAME, ReadWriteIOUtils.readString(body)); - Assert.assertEquals(FILE_LENGTH, ReadWriteIOUtils.readLong(body)); + private static void assertFileSealBody( + final ByteBuffer body, final String expectedFileName, final long expectedFileLength) { + Assert.assertEquals(expectedFileName, ReadWriteIOUtils.readString(body)); + Assert.assertEquals(expectedFileLength, ReadWriteIOUtils.readLong(body)); Assert.assertFalse(body.hasRemaining()); } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java index 85e991e1053c7..9174ec62b2a08 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/request/PipeTransferFileSealReqV2Test.java @@ -27,10 +27,12 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertAirGapReqBytes; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.assertVersionAndType; import static org.apache.iotdb.commons.pipe.sink.payload.thrift.PipeTransferReqTestUtils.copyOf; @@ -59,24 +61,13 @@ public void testSnapshotSealReqV2RoundTripKeepsFilesAndParameters() throws IOExc final List fileLengths = Arrays.asList(12L, 34L); final Map parameters = snapshotParameters(); - final DummyFileSealReqV2 req = - DummyFileSealReqV2.toTPipeTransferReq(fileNames, fileLengths, parameters); - - assertVersionAndType( - req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL); - Assert.assertEquals(fileNames, req.getFileNames()); - Assert.assertEquals(fileLengths, req.getFileLengths()); - Assert.assertEquals(parameters, req.getParameters()); - assertSnapshotSealBody(req.body.duplicate(), fileNames, fileLengths, parameters); - - final DummyFileSealReqV2 deserializedReq = - (DummyFileSealReqV2) new DummyFileSealReqV2().translateFromTPipeTransferReq(copyOf(req)); + assertSnapshotSealReqRoundTrip(fileNames, fileLengths, parameters); + } - Assert.assertEquals(req.version, deserializedReq.version); - Assert.assertEquals(req.type, deserializedReq.type); - Assert.assertEquals(fileNames, deserializedReq.getFileNames()); - Assert.assertEquals(fileLengths, deserializedReq.getFileLengths()); - Assert.assertEquals(parameters, deserializedReq.getParameters()); + @Test + public void testEmptySnapshotSealReqV2RoundTripKeepsFilesAndParameters() throws IOException { + assertSnapshotSealReqRoundTrip( + Collections.emptyList(), Collections.emptyList(), Collections.emptyMap()); } @Test @@ -85,17 +76,39 @@ public void testSnapshotSealAirGapBytesKeepSameBodyFormat() throws IOException { final List fileLengths = Arrays.asList(12L, 34L); final Map parameters = snapshotParameters(); - final ByteBuffer buffer = - ByteBuffer.wrap( - new DummyFileSealReqV2() - .convertToTPipeTransferSnapshotSealBytes(fileNames, fileLengths, parameters)); + assertAirGapReqBytes( + new DummyFileSealReqV2() + .convertToTPipeTransferSnapshotSealBytes(fileNames, fileLengths, parameters), + IoTDBSinkRequestVersion.VERSION_1, + PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL, + body -> assertSnapshotSealBody(body, fileNames, fileLengths, parameters)); + } - Assert.assertEquals( - IoTDBSinkRequestVersion.VERSION_1.getVersion(), ReadWriteIOUtils.readByte(buffer)); - Assert.assertEquals( - PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL.getType(), - ReadWriteIOUtils.readShort(buffer)); - assertSnapshotSealBody(buffer, fileNames, fileLengths, parameters); + private static void assertSnapshotSealReqRoundTrip( + final List expectedFileNames, + final List expectedFileLengths, + final Map expectedParameters) + throws IOException { + final DummyFileSealReqV2 req = + DummyFileSealReqV2.toTPipeTransferReq( + expectedFileNames, expectedFileLengths, expectedParameters); + + assertVersionAndType( + req, IoTDBSinkRequestVersion.VERSION_1, PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL); + Assert.assertEquals(expectedFileNames, req.getFileNames()); + Assert.assertEquals(expectedFileLengths, req.getFileLengths()); + Assert.assertEquals(expectedParameters, req.getParameters()); + assertSnapshotSealBody( + req.body.duplicate(), expectedFileNames, expectedFileLengths, expectedParameters); + + final DummyFileSealReqV2 deserializedReq = + (DummyFileSealReqV2) new DummyFileSealReqV2().translateFromTPipeTransferReq(copyOf(req)); + + Assert.assertEquals(req.version, deserializedReq.version); + Assert.assertEquals(req.type, deserializedReq.type); + Assert.assertEquals(expectedFileNames, deserializedReq.getFileNames()); + Assert.assertEquals(expectedFileLengths, deserializedReq.getFileLengths()); + Assert.assertEquals(expectedParameters, deserializedReq.getParameters()); } private static void assertCapture( @@ -130,13 +143,18 @@ private static void assertSnapshotSealBody( Assert.assertEquals(expectedFileLength.longValue(), ReadWriteIOUtils.readLong(body)); } + final Map parameters = readStringMap(body); + Assert.assertEquals(expectedParameters, parameters); + Assert.assertFalse(body.hasRemaining()); + } + + private static Map readStringMap(final ByteBuffer body) { final int parameterSize = ReadWriteIOUtils.readInt(body); - final Map parameters = new LinkedHashMap<>(); + final Map parameters = new HashMap<>(); for (int i = 0; i < parameterSize; i++) { parameters.put(ReadWriteIOUtils.readString(body), ReadWriteIOUtils.readString(body)); } - Assert.assertEquals(expectedParameters, parameters); - Assert.assertFalse(body.hasRemaining()); + return parameters; } private static Map markerParameters(final String... keys) { From 9fe4aa497dc9d4a4b3b4ee3861bb6c0063271bd3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:31:21 +0800 Subject: [PATCH 5/9] Improve pipe pattern visitor tests --- ...eStatementTreePatternParseVisitorTest.java | 111 +++++++++--------- .../PipePlanTreePatternParseVisitorTest.java | 79 +++++++++++++ 2 files changed, 135 insertions(+), 55 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java index 2dc0a8b9332da..cc0490dd72771 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTreePatternParseVisitorTest.java @@ -46,6 +46,9 @@ public class PipeStatementTreePatternParseVisitorTest { + private final PipeStatementTreePatternParseVisitor visitor = + new PipeStatementTreePatternParseVisitor(); + private final IoTDBTreePatternOperations prefixPathPattern = new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")); private final IoTDBTreePatternOperations fullPathPattern = @@ -55,6 +58,8 @@ public class PipeStatementTreePatternParseVisitorTest { Arrays.asList( new IoTDBTreePattern("root.db.device.s1"), new IoTDBTreePattern("root.db.device.s2"))); + private final IoTDBTreePatternOperations nonOverlappingPattern = + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db1.device.**")); private final IoTDBTreePatternOperations exclusionPattern = new WithExclusionIoTDBTreePattern( new UnionIoTDBTreePattern( @@ -65,68 +70,47 @@ public class PipeStatementTreePatternParseVisitorTest { @Test public void testCreateTimeSeries() throws IllegalPathException { - final CreateTimeSeriesStatement createTimeSeriesStatementS1 = new CreateTimeSeriesStatement(); - createTimeSeriesStatementS1.setPath(new MeasurementPath("root.db.device.s1")); - createTimeSeriesStatementS1.setDataType(TSDataType.FLOAT); - createTimeSeriesStatementS1.setEncoding(TSEncoding.RLE); - createTimeSeriesStatementS1.setCompressor(CompressionType.SNAPPY); - createTimeSeriesStatementS1.setProps(Collections.emptyMap()); - createTimeSeriesStatementS1.setTags(Collections.emptyMap()); - createTimeSeriesStatementS1.setAttributes(Collections.emptyMap()); - createTimeSeriesStatementS1.setAlias("a1"); - - final CreateTimeSeriesStatement createTimeSeriesStatementS2 = new CreateTimeSeriesStatement(); - createTimeSeriesStatementS2.setPath(new MeasurementPath("root.db.device.s2")); - createTimeSeriesStatementS2.setDataType(TSDataType.INT32); - createTimeSeriesStatementS2.setEncoding(TSEncoding.PLAIN); - createTimeSeriesStatementS2.setCompressor(CompressionType.SNAPPY); - createTimeSeriesStatementS2.setProps(Collections.emptyMap()); - createTimeSeriesStatementS2.setTags(Collections.emptyMap()); - createTimeSeriesStatementS2.setAttributes(Collections.emptyMap()); - createTimeSeriesStatementS2.setAlias("a2"); - + final CreateTimeSeriesStatement createTimeSeriesStatementS1 = + createTimeSeriesStatement("root.db.device.s1", TSDataType.FLOAT, TSEncoding.RLE, "a1"); + final CreateTimeSeriesStatement createTimeSeriesStatementS2 = + createTimeSeriesStatement("root.db.device.s2", TSDataType.INT32, TSEncoding.PLAIN, "a2"); final CreateTimeSeriesStatement createTimeSeriesStatementToFilter = - new CreateTimeSeriesStatement(); - createTimeSeriesStatementToFilter.setPath(new MeasurementPath("root.db1.device.s1")); - createTimeSeriesStatementToFilter.setDataType(TSDataType.FLOAT); - createTimeSeriesStatementToFilter.setEncoding(TSEncoding.RLE); - createTimeSeriesStatementToFilter.setCompressor(CompressionType.SNAPPY); - createTimeSeriesStatementToFilter.setProps(Collections.emptyMap()); - createTimeSeriesStatementToFilter.setTags(Collections.emptyMap()); - createTimeSeriesStatementToFilter.setAttributes(Collections.emptyMap()); - createTimeSeriesStatementToFilter.setAlias("a3"); + createTimeSeriesStatement("root.db1.device.s1", TSDataType.FLOAT, TSEncoding.RLE, "a3"); Assert.assertEquals( createTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementS1, prefixPathPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); Assert.assertEquals( createTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementS1, multiplePathPattern) .orElseThrow(AssertionError::new)); + Assert.assertEquals( + createTimeSeriesStatementS2, + visitor + .visitCreateTimeseries(createTimeSeriesStatementS2, multiplePathPattern) + .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementToFilter, multiplePathPattern) .isPresent()); Assert.assertEquals( createTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementS1, exclusionPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() - .visitCreateTimeseries(createTimeSeriesStatementS2, exclusionPattern) - .isPresent()); + visitor.visitCreateTimeseries(createTimeSeriesStatementS2, exclusionPattern).isPresent()); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateTimeseries(createTimeSeriesStatementToFilter, exclusionPattern) .isPresent()); } @@ -169,13 +153,13 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { Assert.assertEquals( expectedS1Only, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateAlignedTimeseries(originalS1S2, fullPathPattern) .orElseThrow(AssertionError::new)); Assert.assertEquals( originalS1S2, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateAlignedTimeseries(originalS1S2S3, multiplePathPattern) .orElseThrow(AssertionError::new)); @@ -191,9 +175,11 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { Assert.assertEquals( expectedS1S3, - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateAlignedTimeseries(originalS1S2S3, exclusionPattern) .orElseThrow(AssertionError::new)); + Assert.assertFalse( + visitor.visitCreateAlignedTimeseries(originalS1S2S3, nonOverlappingPattern).isPresent()); } @Test @@ -224,35 +210,33 @@ public void testAlterTimeSeries() throws IllegalPathException { Assert.assertEquals( alterTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementS1, fullPathPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, prefixPathPattern) .isPresent()); Assert.assertEquals( alterTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementS1, multiplePathPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, multiplePathPattern) .isPresent()); Assert.assertEquals( alterTimeSeriesStatementS1, - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementS1, exclusionPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() - .visitAlterTimeSeries(alterTimeSeriesStatementS2, exclusionPattern) - .isPresent()); + visitor.visitAlterTimeSeries(alterTimeSeriesStatementS2, exclusionPattern).isPresent()); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitAlterTimeSeries(alterTimeSeriesStatementToFilter, exclusionPattern) .isPresent()); } @@ -266,20 +250,20 @@ public void testActivateTemplate() throws IllegalPathException { Assert.assertEquals( activateTemplateStatement, - new PipeStatementTreePatternParseVisitor() + visitor .visitActivateTemplate(activateTemplateStatement, prefixPathPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitActivateTemplate(activateTemplateStatementToFilter, prefixPathPattern) .isPresent()); Assert.assertEquals( activateTemplateStatement, - new PipeStatementTreePatternParseVisitor() + visitor .visitActivateTemplate(activateTemplateStatement, multiplePathPattern) .orElseThrow(AssertionError::new)); Assert.assertFalse( - new PipeStatementTreePatternParseVisitor() + visitor .visitActivateTemplate(activateTemplateStatementToFilter, multiplePathPattern) .isPresent()); } @@ -295,7 +279,7 @@ public void testCreateLogicalView() throws IllegalPathException { final CreateLogicalViewStatement targetLogicalViewStatement = (CreateLogicalViewStatement) - new PipeStatementTreePatternParseVisitor() + visitor .visitCreateLogicalView(createLogicalViewStatement, prefixPathPattern) .orElseThrow(AssertionError::new); Assert.assertEquals( @@ -304,5 +288,22 @@ public void testCreateLogicalView() throws IllegalPathException { Assert.assertEquals( Collections.singletonList(new TimeSeriesViewOperand("root.sg1.d1")), targetLogicalViewStatement.getViewExpressions()); + Assert.assertFalse( + visitor.visitCreateLogicalView(createLogicalViewStatement, fullPathPattern).isPresent()); + } + + private static CreateTimeSeriesStatement createTimeSeriesStatement( + final String path, final TSDataType dataType, final TSEncoding encoding, final String alias) + throws IllegalPathException { + final CreateTimeSeriesStatement statement = new CreateTimeSeriesStatement(); + statement.setPath(new MeasurementPath(path)); + statement.setDataType(dataType); + statement.setEncoding(encoding); + statement.setCompressor(CompressionType.SNAPPY); + statement.setProps(Collections.emptyMap()); + statement.setTags(Collections.emptyMap()); + statement.setAttributes(Collections.emptyMap()); + statement.setAlias(alias); + return statement; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java index ea071c3dcaff4..4704834077af0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTreePatternParseVisitorTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; +import org.apache.iotdb.commons.pipe.datastructure.pattern.WithExclusionIoTDBTreePattern; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.schema.view.viewExpression.leaf.TimeSeriesViewOperand; @@ -67,6 +68,10 @@ public class PipePlanTreePatternParseVisitorTest { Arrays.asList( new IoTDBTreePattern("root.db.device.s1"), new IoTDBTreePattern("root.db.device.s2"))); + private final IoTDBTreePatternOperations exclusionPattern = + new WithExclusionIoTDBTreePattern( + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.**")), + new UnionIoTDBTreePattern(new IoTDBTreePattern("root.db.device.s2"))); @Test public void testCreateTimeSeries() throws IllegalPathException { @@ -186,6 +191,50 @@ public void testCreateAlignedTimeSeries() throws IllegalPathException { Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), multiplePathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s3"), + Arrays.asList(TSDataType.FLOAT, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.RLE), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a3"), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList(Collections.emptyMap(), Collections.emptyMap())), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateAlignedTimeSeries( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-3"), + new PartialPath("root.db.device"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList(TSDataType.FLOAT, TSDataType.BOOLEAN, TSDataType.INT32), + Arrays.asList(TSEncoding.RLE, TSEncoding.PLAIN, TSEncoding.RLE), + Arrays.asList( + CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.SNAPPY), + Arrays.asList("a1", "a2", "a3"), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()), + Arrays.asList( + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap())), + exclusionPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitCreateAlignedTimeSeries( + new CreateAlignedTimeSeriesNode( + new PlanNodeId("2024-04-30-4"), + new PartialPath("root.db.device"), + Collections.singletonList("s2"), + Collections.singletonList(TSDataType.BOOLEAN), + Collections.singletonList(TSEncoding.PLAIN), + Collections.singletonList(CompressionType.SNAPPY), + Collections.singletonList("a2"), + Collections.singletonList(Collections.emptyMap()), + Collections.singletonList(Collections.emptyMap())), + exclusionPattern) + .isPresent()); } @Test @@ -635,5 +684,35 @@ public void testDeleteData() throws IllegalPathException { Long.MAX_VALUE), multiplePathPattern) .orElseThrow(AssertionError::new)); + + Assert.assertEquals( + new DeleteDataNode( + new PlanNodeId("2024-04-30-3"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), new MeasurementPath("root.db.device.s3")), + Long.MIN_VALUE, + Long.MAX_VALUE), + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteData( + new DeleteDataNode( + new PlanNodeId("2024-04-30-3"), + Arrays.asList( + new MeasurementPath("root.db.device.s1"), + new MeasurementPath("root.db.device.s2"), + new MeasurementPath("root.db.device.s3")), + Long.MIN_VALUE, + Long.MAX_VALUE), + exclusionPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBSchemaRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitDeleteData( + new DeleteDataNode( + new PlanNodeId("2024-04-30-4"), + Collections.singletonList(new MeasurementPath("root.db.device.s2")), + Long.MIN_VALUE, + Long.MAX_VALUE), + exclusionPattern) + .isPresent()); } } From 97ac798a59c9037d0586fe9c55a46862372b4f19 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:15:36 +0800 Subject: [PATCH 6/9] Improve pipe visitor tests --- .../PipeConfigScopeParseVisitorTest.java | 94 +++--- ...ipeConfigTablePatternParseVisitorTest.java | 299 +++++++----------- ...PipeConfigTreePatternParseVisitorTest.java | 242 ++++++-------- ...StatementTablePatternParseVisitorTest.java | 46 ++- .../PipePlanTablePatternParseVisitorTest.java | 142 ++++----- 5 files changed, 358 insertions(+), 465 deletions(-) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java index 67887d58a0737..a6aa1e2a1f16a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigScopeParseVisitorTest.java @@ -30,7 +30,9 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; public class PipeConfigScopeParseVisitorTest { @@ -44,38 +46,35 @@ public void testTreeScopeParsing() { private void testTreeScopeParsing(final ConfigPhysicalPlanType type, final boolean isUser) { Assert.assertEquals( - new AuthorTreePlan( - type, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(PrivilegeType::forRelationalSys) - .map(Enum::ordinal) - .collect(Collectors.toList())), - false, - Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), + treeAuthorPlan( + type, isUser, privileges(PrivilegeType::forRelationalSys), false, rootPattern()), IoTDBConfigRegionSource.TREE_SCOPE_PARSE_VISITOR .process( - new AuthorTreePlan( + treeAuthorPlan( type, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(privilegeType -> !privilegeType.isRelationalPrivilege()) - .map(Enum::ordinal) - .collect(Collectors.toList())), + isUser, + privileges(privilegeType -> !privilegeType.isRelationalPrivilege()), false, - Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), + rootPattern()), null) .orElseThrow(AssertionError::new)); } + @Test + public void testTreeScopeParsingSkipsWithoutRelationalSystemPrivilege() { + Assert.assertFalse( + IoTDBConfigRegionSource.TREE_SCOPE_PARSE_VISITOR + .process( + treeAuthorPlan( + ConfigPhysicalPlanType.GrantUser, + true, + Collections.singleton(PrivilegeType.READ_DATA.ordinal()), + false, + rootPattern()), + null) + .isPresent()); + } + @Test public void testTableScopeParsing() { testTableScopeParsing( @@ -93,24 +92,41 @@ private void testTableScopeParsing( final ConfigPhysicalPlanType inputType, final boolean isUser) { Assert.assertEquals( - new AuthorTreePlan( + treeAuthorPlan( outputType, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(PrivilegeType::forRelationalSys) - .map(Enum::ordinal) - .collect(Collectors.toList())), + isUser, + privileges(PrivilegeType::forRelationalSys), true, Collections.emptyList()), IoTDBConfigRegionSource.TABLE_SCOPE_PARSE_VISITOR - .process( - new AuthorRelationalPlan( - inputType, isUser ? "user" : "", isUser ? "" : "role", "", "", -1, true), - null) + .process(relationalAuthorPlan(inputType, isUser, true), null) .orElseThrow(AssertionError::new)); } + + private AuthorTreePlan treeAuthorPlan( + final ConfigPhysicalPlanType type, + final boolean isUser, + final Set permissions, + final boolean grantOption, + final List paths) { + return new AuthorTreePlan( + type, isUser ? "user" : "", isUser ? "" : "role", "", "", permissions, grantOption, paths); + } + + private AuthorRelationalPlan relationalAuthorPlan( + final ConfigPhysicalPlanType type, final boolean isUser, final boolean grantOption) { + return new AuthorRelationalPlan( + type, isUser ? "user" : "", isUser ? "" : "role", "", "", -1, grantOption); + } + + private Set privileges(final Predicate predicate) { + return Arrays.stream(PrivilegeType.values()) + .filter(predicate) + .map(Enum::ordinal) + .collect(Collectors.toSet()); + } + + private List rootPattern() { + return Collections.singletonList(new PartialPath(new String[] {"root", "**"})); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java index 229c0452dcc0a..174f0e395546a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTablePatternParseVisitorTest.java @@ -51,218 +51,147 @@ import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; public class PipeConfigTablePatternParseVisitorTest { - private final TablePattern tablePattern = new TablePattern(true, "^db[0-9]", "a.*b"); - - @Test - public void testCreateDatabase() { - testInput( - new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("db1")), - new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("da")), - new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("dc"))); - } - - @Test - public void testAlterDatabase() { - testInput( - new DatabaseSchemaPlan(ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema("db1")), - new DatabaseSchemaPlan(ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema("da")), - new DatabaseSchemaPlan(ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema("dc"))); - } - @Test - public void testDeleteDatabase() { - testInput( - new DeleteDatabasePlan("db1"), new DeleteDatabasePlan("da"), new DeleteDatabasePlan("dc")); - } - - @Test - public void testCreateTableOrView() { - testInput( - new PipeCreateTableOrViewPlan("db1", new TsTable("ab")), - new PipeCreateTableOrViewPlan("db1", new TsTable("ac")), - new PipeCreateTableOrViewPlan("da", new TsTable("a2b"))); - } - - @Test - public void testAddTableColumn() { - testInput( - new AddTableColumnPlan("db1", "ab", new ArrayList<>(), false), - new AddTableColumnPlan("db1", "ac", new ArrayList<>(), false), - new AddTableColumnPlan("da", "ac", new ArrayList<>(), false)); - } - - @Test - public void testAddViewColumn() { - testInput( - new AddTableViewColumnPlan("db1", "ab", new ArrayList<>(), false), - new AddTableViewColumnPlan("db1", "ac", new ArrayList<>(), false), - new AddTableViewColumnPlan("da", "ac", new ArrayList<>(), false)); - } - - @Test - public void testSetTableProperties() { - testInput( - new SetTablePropertiesPlan("db1", "ab", Collections.singletonMap("ttl", "2")), - new SetTablePropertiesPlan("db1", "ac", Collections.singletonMap("ttl", "2")), - new SetTablePropertiesPlan("da", "ac", Collections.singletonMap("ttl", "2"))); - } - - @Test - public void testSetViewProperties() { - testInput( - new SetViewPropertiesPlan("db1", "ab", Collections.singletonMap("ttl", "2")), - new SetViewPropertiesPlan("db1", "ac", Collections.singletonMap("ttl", "2")), - new SetViewPropertiesPlan("da", "ac", Collections.singletonMap("ttl", "2"))); - } - - @Test - public void testCommitDeleteColumn() { - testInput( - new CommitDeleteColumnPlan("db1", "ab", "a"), - new CommitDeleteColumnPlan("db1", "ac", "a"), - new CommitDeleteColumnPlan("da", "ac", "a")); - } + private static final String MATCH_DATABASE = "db1"; + private static final String QUALIFIED_MATCH_DATABASE = "root.db1"; + private static final String MISMATCH_DATABASE = "da"; + private static final String MATCH_TABLE = "ab"; + private static final String MISMATCH_TABLE = "ac"; - @Test - public void testCommitDeleteViewColumn() { - testInput( - new CommitDeleteViewColumnPlan("db1", "ab", "a"), - new CommitDeleteViewColumnPlan("db1", "ac", "a"), - new CommitDeleteViewColumnPlan("da", "ac", "a")); - } - - @Test - public void testRenameTableColumn() { - testInput( - new RenameTableColumnPlan("db1", "ab", "old", "new"), - new RenameTableColumnPlan("db1", "ac", "old", "new"), - new RenameTableColumnPlan("da", "ac", "old", "new")); - } - - @Test - public void testRenameViewColumn() { - testInput( - new RenameViewColumnPlan("db1", "ab", "old", "new"), - new RenameViewColumnPlan("db1", "ac", "old", "new"), - new RenameViewColumnPlan("da", "ac", "old", "new")); - } + private final TablePattern tablePattern = new TablePattern(true, "^db[0-9]", "a.*b"); @Test - public void testCommitDeleteTable() { - testInput( - new CommitDeleteTablePlan("db1", "ab"), - new CommitDeleteTablePlan("db1", "ac"), - new CommitDeleteTablePlan("da", "ac")); - } + public void testDatabasePlans() { + final List> planFactories = + Arrays.asList( + database -> + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database)), + database -> + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema(database)), + DeleteDatabasePlan::new); + + planFactories.forEach(this::assertDatabasePlanMatches); + } + + @Test + public void testTablePlans() { + final List> planFactories = + Arrays.asList( + (database, table) -> new PipeCreateTableOrViewPlan(database, new TsTable(table)), + (database, table) -> + new AddTableColumnPlan(database, table, Collections.emptyList(), false), + (database, table) -> + new AddTableViewColumnPlan(database, table, Collections.emptyList(), false), + (database, table) -> + new SetTablePropertiesPlan(database, table, Collections.singletonMap("ttl", "2")), + (database, table) -> + new SetViewPropertiesPlan(database, table, Collections.singletonMap("ttl", "2")), + (database, table) -> new CommitDeleteColumnPlan(database, table, "a"), + (database, table) -> new CommitDeleteViewColumnPlan(database, table, "a"), + (database, table) -> new RenameTableColumnPlan(database, table, "old", "new"), + (database, table) -> new RenameViewColumnPlan(database, table, "old", "new"), + CommitDeleteTablePlan::new, + CommitDeleteViewPlan::new, + (database, table) -> + new PipeDeleteDevicesPlan(database, table, new byte[0], new byte[0], new byte[0]), + (database, table) -> new SetTableCommentPlan(database, table, "a"), + (database, table) -> new SetViewCommentPlan(database, table, "a"), + (database, table) -> new SetTableColumnCommentPlan(database, table, "a", "a"), + (database, table) -> new RenameTablePlan(database, table, MISMATCH_TABLE), + (database, table) -> new RenameViewPlan(database, table, MISMATCH_TABLE), + (database, table) -> + new AlterColumnDataTypePlan(database, table, "a", TSDataType.INT64)); + + planFactories.forEach(this::assertTablePlanMatches); + } + + @Test + public void testAuthorDatabasePlans() { + final List planTypes = + Arrays.asList( + ConfigPhysicalPlanType.RGrantUserDBPriv, + ConfigPhysicalPlanType.RGrantRoleDBPriv, + ConfigPhysicalPlanType.RRevokeUserDBPriv, + ConfigPhysicalPlanType.RRevokeRoleDBPriv); - @Test - public void testCommitDeleteView() { - testInput( - new CommitDeleteViewPlan("db1", "ab"), - new CommitDeleteViewPlan("db1", "ac"), - new CommitDeleteViewPlan("da", "ac")); + planTypes.forEach(this::assertAuthorDatabasePlanMatches); } - // Match the oldName instead of the new one @Test - public void testRenameTable() { - testInput( - new RenameTablePlan("db1", "ab", "ac"), - new RenameTablePlan("db1", "ac", "ab"), - new RenameTablePlan("da", "ac", "ab")); - } + public void testAuthorTablePlans() { + final List planTypes = + Arrays.asList( + ConfigPhysicalPlanType.RGrantUserTBPriv, + ConfigPhysicalPlanType.RGrantRoleTBPriv, + ConfigPhysicalPlanType.RRevokeUserTBPriv, + ConfigPhysicalPlanType.RRevokeRoleTBPriv); - @Test - public void testRenameView() { - testInput( - new RenameViewPlan("db1", "ab", "ac"), - new RenameViewPlan("db1", "ac", "ab"), - new RenameViewPlan("da", "ac", "ab")); + planTypes.forEach(this::assertAuthorTablePlanMatches); } @Test - public void testPipeDeleteDevices() { - testInput( - new PipeDeleteDevicesPlan("db1", "ab", new byte[0], new byte[0], new byte[0]), - new PipeDeleteDevicesPlan("db1", "ac", new byte[0], new byte[0], new byte[0]), - new PipeDeleteDevicesPlan("da", "ac", new byte[0], new byte[0], new byte[0])); + public void testUnscopedAuthorPlanPassesThrough() { + assertPatternMatches( + authorPlan(ConfigPhysicalPlanType.RGrantRoleAll, "", "", PrivilegeType.SELECT)); } - @Test - public void testSetTableComment() { - testInput( - new SetTableCommentPlan("db1", "ab", "a"), - new SetTableCommentPlan("db1", "ac", "a"), - new SetTableCommentPlan("da", "ac", "a")); + private void assertDatabasePlanMatches(final Function planFactory) { + assertPatternMatches( + planFactory.apply(QUALIFIED_MATCH_DATABASE), planFactory.apply(MISMATCH_DATABASE)); } - @Test - public void testSetViewComment() { - testInput( - new SetViewCommentPlan("db1", "ab", "a"), - new SetViewCommentPlan("db1", "ac", "a"), - new SetViewCommentPlan("da", "ac", "a")); + private void assertTablePlanMatches( + final BiFunction planFactory) { + assertPatternMatches( + planFactory.apply(QUALIFIED_MATCH_DATABASE, MATCH_TABLE), + planFactory.apply(QUALIFIED_MATCH_DATABASE, MISMATCH_TABLE), + planFactory.apply(MISMATCH_DATABASE, MATCH_TABLE)); } - @Test - public void testSetTableColumnComment() { - testInput( - new SetTableColumnCommentPlan("db1", "ab", "a", "a"), - new SetTableColumnCommentPlan("db1", "ac", "a", "a"), - new SetTableColumnCommentPlan("da", "ac", "a", "a")); + private void assertAuthorDatabasePlanMatches(final ConfigPhysicalPlanType type) { + assertPatternMatches( + authorPlan(type, MATCH_DATABASE, "", PrivilegeType.SELECT), + authorPlan(type, MISMATCH_DATABASE, "", PrivilegeType.SELECT)); } - @Test - public void testAuth() { - testInput( - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantRoleAll, "", "role", "", "", -1, false), - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantUserDBPriv, - "user", - "", - "da", - "", - PrivilegeType.SELECT.ordinal(), - false), - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantUserTBPriv, - "user", - "", - "db1", - "ac", - PrivilegeType.DROP.ordinal(), - false)); + private void assertAuthorTablePlanMatches(final ConfigPhysicalPlanType type) { + assertPatternMatches( + authorPlan(type, MATCH_DATABASE, MATCH_TABLE, PrivilegeType.SELECT), + authorPlan(type, MATCH_DATABASE, MISMATCH_TABLE, PrivilegeType.SELECT), + authorPlan(type, MISMATCH_DATABASE, MATCH_TABLE, PrivilegeType.SELECT)); } - private void testInput( - final ConfigPhysicalPlan trueInput, - final ConfigPhysicalPlan falseInput1, - final ConfigPhysicalPlan falseInput2) { + private void assertPatternMatches( + final ConfigPhysicalPlan matchedInput, final ConfigPhysicalPlan... filteredInputs) { Assert.assertEquals( - trueInput, + matchedInput, IoTDBConfigRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(trueInput, tablePattern) + .process(matchedInput, tablePattern) .orElseThrow(AssertionError::new)); - Assert.assertFalse( - IoTDBConfigRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(falseInput1, tablePattern) - .isPresent()); - Assert.assertFalse( - IoTDBConfigRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(falseInput2, tablePattern) - .isPresent()); - } - - @Test - public void testAlterTableColumnDataType() { - testInput( - new AlterColumnDataTypePlan("db1", "ab", "a", TSDataType.INT64), - new AlterColumnDataTypePlan("db1", "ac", "a", TSDataType.BLOB), - new AlterColumnDataTypePlan("da", "ac", "a", TSDataType.DATE)); + Arrays.stream(filteredInputs) + .forEach( + filteredInput -> + Assert.assertFalse( + IoTDBConfigRegionSource.TABLE_PATTERN_PARSE_VISITOR + .process(filteredInput, tablePattern) + .isPresent())); + } + + private AuthorRelationalPlan authorPlan( + final ConfigPhysicalPlanType type, + final String database, + final String table, + final PrivilegeType privilegeType) { + return new AuthorRelationalPlan( + type, "user", "role", database, table, privilegeType.ordinal(), false); } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java index cdd2a1f484067..374a7fb1b88b9 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigTreePatternParseVisitorTest.java @@ -27,11 +27,13 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; import org.apache.iotdb.commons.pipe.datastructure.pattern.UnionIoTDBTreePattern; import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterTimeSeriesPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan; @@ -55,6 +57,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.function.Function; public class PipeConfigTreePatternParseVisitorTest { @@ -69,74 +72,16 @@ public class PipeConfigTreePatternParseVisitorTest { new IoTDBTreePattern("root.db.device.s2"))); @Test - public void testCreateDatabase() { - final DatabaseSchemaPlan createDatabasePlan = - new DatabaseSchemaPlan( - ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("root.db")); - final DatabaseSchemaPlan createDatabasePlanToFilter = - new DatabaseSchemaPlan( - ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("root.db1")); - - Assert.assertEquals( - createDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitCreateDatabase(createDatabasePlan, prefixPathPattern) - .orElseThrow(AssertionError::new)); - Assert.assertFalse( - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitCreateDatabase(createDatabasePlanToFilter, prefixPathPattern) - .isPresent()); - Assert.assertEquals( - createDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitCreateDatabase(createDatabasePlan, multiplePathPattern) - .orElseThrow(AssertionError::new)); - } - - @Test - public void testAlterDatabase() { - final DatabaseSchemaPlan alterDatabasePlan = - new DatabaseSchemaPlan( - ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema("root.db")); - final DatabaseSchemaPlan alterDatabasePlanToFilter = - new DatabaseSchemaPlan( - ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema("root.db1")); - - Assert.assertEquals( - alterDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitAlterDatabase(alterDatabasePlan, prefixPathPattern) - .orElseThrow(AssertionError::new)); - Assert.assertFalse( - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitAlterDatabase(alterDatabasePlanToFilter, prefixPathPattern) - .isPresent()); - Assert.assertEquals( - alterDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitAlterDatabase(alterDatabasePlan, multiplePathPattern) - .orElseThrow(AssertionError::new)); - } - - @Test - public void testDeleteDatabase() { - final DeleteDatabasePlan deleteDatabasePlan = new DeleteDatabasePlan("root.db"); - final DeleteDatabasePlan deleteDatabasePlanToFilter = new DeleteDatabasePlan("root.db1"); - - Assert.assertEquals( - deleteDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitDeleteDatabase(deleteDatabasePlan, prefixPathPattern) - .orElseThrow(AssertionError::new)); - Assert.assertFalse( - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitDeleteDatabase(deleteDatabasePlanToFilter, prefixPathPattern) - .isPresent()); - Assert.assertEquals( - deleteDatabasePlan, - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitDeleteDatabase(deleteDatabasePlan, multiplePathPattern) - .orElseThrow(AssertionError::new)); + public void testDatabasePlans() { + Arrays.>asList( + database -> + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database)), + database -> + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.AlterDatabase, new TDatabaseSchema(database)), + DeleteDatabasePlan::new) + .forEach(this::assertDatabasePlanMatches); } @Test @@ -270,91 +215,11 @@ public void testExtendSchemaTemplate() { } @Test - public void testGrantUser() throws IllegalPathException { - Assert.assertEquals( - Collections.singletonList(new PartialPath("root.db.device.**")), - ((AuthorTreePlan) - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitGrantUser( - new AuthorTreePlan( - ConfigPhysicalPlanType.GrantUser, - "tempUser", - "", - "", - "", - new HashSet<>(Arrays.asList(1, 2)), - true, - Arrays.asList( - new PartialPath("root.db.**"), new PartialPath("root.abc.**"))), - prefixPathPattern) - .orElseThrow(AssertionError::new)) - .getNodeNameList()); - } - - @Test - public void testRevokeUser() throws IllegalPathException { - Assert.assertEquals( - Collections.singletonList(new PartialPath("root.db.device.**")), - ((AuthorTreePlan) - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitRevokeUser( - new AuthorTreePlan( - ConfigPhysicalPlanType.RevokeUser, - "tempUser", - "", - "", - "", - new HashSet<>(Arrays.asList(1, 2)), - false, - Arrays.asList( - new PartialPath("root.db.**"), new PartialPath("root.abc.**"))), - prefixPathPattern) - .orElseThrow(AssertionError::new)) - .getNodeNameList()); - } - - @Test - public void testGrantRole() throws IllegalPathException { - Assert.assertEquals( - Collections.singletonList(new PartialPath("root.db.device.**")), - ((AuthorTreePlan) - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitGrantRole( - new AuthorTreePlan( - ConfigPhysicalPlanType.GrantRole, - "", - "tempRole", - "", - "", - new HashSet<>(Arrays.asList(1, 2)), - true, - Arrays.asList( - new PartialPath("root.db.**"), new PartialPath("root.abc.**"))), - prefixPathPattern) - .orElseThrow(AssertionError::new)) - .getNodeNameList()); - } - - @Test - public void testRevokeRole() throws IllegalPathException { - Assert.assertEquals( - Collections.singletonList(new PartialPath("root.db.device.**")), - ((AuthorTreePlan) - IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR - .visitRevokeRole( - new AuthorTreePlan( - ConfigPhysicalPlanType.RevokeRole, - "", - "tempRole", - "", - "", - new HashSet<>(Arrays.asList(1, 2)), - false, - Arrays.asList( - new PartialPath("root.db.**"), new PartialPath("root.abc.**"))), - prefixPathPattern) - .orElseThrow(AssertionError::new)) - .getNodeNameList()); + public void testAuthorPlans() throws IllegalPathException { + assertAuthorPlanMatches(ConfigPhysicalPlanType.GrantUser, true, true); + assertAuthorPlanMatches(ConfigPhysicalPlanType.RevokeUser, true, false); + assertAuthorPlanMatches(ConfigPhysicalPlanType.GrantRole, false, true); + assertAuthorPlanMatches(ConfigPhysicalPlanType.RevokeRole, false, false); } @Test @@ -419,6 +284,29 @@ public void testPipeDeleteLogicalView() throws IllegalPathException, IOException .getAllPathPatterns()); } + @Test + public void testPipeAlterEncodingCompressor() throws IllegalPathException, IOException { + final PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(new PartialPath("root.*.device.s1")); + patternTree.constructTree(); + + final PipeAlterEncodingCompressorPlan plan = + (PipeAlterEncodingCompressorPlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .visitPipeAlterEncodingCompressor( + new PipeAlterEncodingCompressorPlan( + patternTree.serialize(), (byte) 1, (byte) 2, true), + prefixPathPattern) + .orElseThrow(AssertionError::new); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.s1")), + PathPatternTree.deserialize(plan.getPatternTreeBytes()).getAllPathPatterns()); + Assert.assertEquals((byte) 1, plan.getEncoding()); + Assert.assertEquals((byte) 2, plan.getCompressor()); + Assert.assertTrue(plan.isMayAlterAudit()); + } + @Test public void testPipeDeactivateTemplate() throws IllegalPathException { final Template template1 = newSchemaTemplate("template1"); @@ -494,4 +382,52 @@ public void testPipeAlterTimeSeries() throws IllegalPathException { Assert.assertEquals((byte) 0, plan.getOperationType()); Assert.assertEquals(plan.getDataType(), TSDataType.DOUBLE); } + + private void assertDatabasePlanMatches(final Function planFactory) { + final ConfigPhysicalPlan matchedPlan = planFactory.apply("root.db"); + final ConfigPhysicalPlan filteredPlan = planFactory.apply("root.db1"); + + Assert.assertEquals( + matchedPlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .process(matchedPlan, prefixPathPattern) + .orElseThrow(AssertionError::new)); + Assert.assertFalse( + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .process(filteredPlan, prefixPathPattern) + .isPresent()); + Assert.assertEquals( + matchedPlan, + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .process(matchedPlan, multiplePathPattern) + .orElseThrow(AssertionError::new)); + } + + private void assertAuthorPlanMatches( + final ConfigPhysicalPlanType type, final boolean isUser, final boolean grantOpt) + throws IllegalPathException { + final AuthorTreePlan plan = + new AuthorTreePlan( + type, + isUser ? "tempUser" : "", + isUser ? "" : "tempRole", + "", + "", + new HashSet<>(Arrays.asList(1, 2)), + grantOpt, + Arrays.asList(new PartialPath("root.db.**"), new PartialPath("root.abc.**"))); + + final AuthorTreePlan parsedPlan = + (AuthorTreePlan) + IoTDBConfigRegionSource.TREE_PATTERN_PARSE_VISITOR + .process(plan, prefixPathPattern) + .orElseThrow(AssertionError::new); + + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.db.device.**")), + parsedPlan.getNodeNameList()); + Assert.assertEquals(plan.getPermissions(), parsedPlan.getPermissions()); + Assert.assertEquals(grantOpt, parsedPlan.getGrantOpt()); + Assert.assertEquals(type, parsedPlan.getAuthorType()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTablePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTablePatternParseVisitorTest.java index 4c040ad4719a5..560e2a599a002 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTablePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementTablePatternParseVisitorTest.java @@ -30,28 +30,42 @@ public class PipeStatementTablePatternParseVisitorTest { + private static final String MATCH_DATABASE = "db1"; + private static final String MISMATCH_DATABASE = "da"; + private static final String MATCH_TABLE = "ab"; + private static final String MISMATCH_TABLE = "ac"; + private final TablePattern tablePattern = new TablePattern(true, "^db[0-9]", "a.*b"); + private final PipeStatementTablePatternParseVisitor visitor = + new PipeStatementTablePatternParseVisitor(); @Test public void testCreateOrUpdateDevice() { - final CreateOrUpdateDevice trueInput = - new CreateOrUpdateDevice( - "db1", "ab", Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - final CreateOrUpdateDevice falseInput1 = - new CreateOrUpdateDevice( - "db1", "ac", Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - final CreateOrUpdateDevice falseInput2 = - new CreateOrUpdateDevice( - "da", "a2b", Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + final CreateOrUpdateDevice matchedInput = createOrUpdateDevice(MATCH_DATABASE, MATCH_TABLE); + + Assert.assertEquals( + matchedInput, visitor.process(matchedInput, tablePattern).orElseThrow(AssertionError::new)); + assertPatternFilters(createOrUpdateDevice(MATCH_DATABASE, MISMATCH_TABLE)); + assertPatternFilters(createOrUpdateDevice(MISMATCH_DATABASE, MATCH_TABLE)); + } + + @Test + public void testCreateOrUpdateDeviceMatchesDefaultPattern() { + final CreateOrUpdateDevice input = createOrUpdateDevice(MISMATCH_DATABASE, MISMATCH_TABLE); Assert.assertEquals( - trueInput, - new PipeStatementTablePatternParseVisitor() - .process(trueInput, tablePattern) + input, + visitor + .process(input, new TablePattern(true, null, null)) .orElseThrow(AssertionError::new)); - Assert.assertFalse( - new PipeStatementTablePatternParseVisitor().process(falseInput1, tablePattern).isPresent()); - Assert.assertFalse( - new PipeStatementTablePatternParseVisitor().process(falseInput2, tablePattern).isPresent()); + } + + private void assertPatternFilters(final CreateOrUpdateDevice input) { + Assert.assertFalse(visitor.process(input, tablePattern).isPresent()); + } + + private CreateOrUpdateDevice createOrUpdateDevice(final String database, final String table) { + return new CreateOrUpdateDevice( + database, table, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java index 0a7fbfb699dc9..9792dacb85a29 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.source; -import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId; @@ -27,9 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.tsfile.read.common.TimeRange; @@ -38,92 +35,93 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; public class PipePlanTablePatternParseVisitorTest { + + private static final String MATCH_DATABASE = "db1"; + private static final String MISMATCH_DATABASE = "da"; + private static final String MATCH_TABLE = "ab"; + private static final String MISMATCH_TABLE = "ac"; + private final TablePattern tablePattern = new TablePattern(true, "^db[0-9]", "a.*b"); @Test - public void testCreateOrUpdateTableDevice() { - testInput( - new CreateOrUpdateTableDeviceNode( - new PlanNodeId(""), - "db1", - "ab", - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList()), - new CreateOrUpdateTableDeviceNode( - new PlanNodeId(""), - "db1", - "ac", - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList()), - new CreateOrUpdateTableDeviceNode( - new PlanNodeId(""), - "da", - "a2b", - Collections.emptyList(), - Collections.emptyList(), - Collections.emptyList())); + public void testTableDevicePlans() { + final List> planFactories = + Arrays.asList(this::createOrUpdateTableDeviceNode, this::tableDeviceAttributeUpdateNode); + + planFactories.forEach(this::assertTableDevicePlanMatches); } @Test - public void testTableDeviceAttributeUpdate() { - testInput( - new TableDeviceAttributeUpdateNode( - new PlanNodeId(""), "db1", "ab", null, null, null, null, null, null), - new TableDeviceAttributeUpdateNode( - new PlanNodeId(""), "db1", "ac", null, null, null, null, null, null), - new TableDeviceAttributeUpdateNode( - new PlanNodeId(""), "da", "a2b", null, null, null, null, null, null)); + public void testDeleteData() { + final TableDeletionEntry matchedEntry = deletionEntry(MATCH_TABLE); + final TableDeletionEntry filteredEntry = deletionEntry(MISMATCH_TABLE); + + Assert.assertEquals( + deleteDataNode(MATCH_DATABASE, matchedEntry), + IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR + .process(deleteDataNode(MATCH_DATABASE, matchedEntry, filteredEntry), tablePattern) + .orElse(null)); } - private void testInput( - final PlanNode trueInput, final PlanNode falseInput1, final PlanNode falseInput2) { + @Test + public void testDeleteDataIsFilteredWhenDatabaseOrTableDoesNotMatch() { + assertPatternFilters(deleteDataNode(null, deletionEntry(MATCH_TABLE))); + assertPatternFilters(deleteDataNode(MISMATCH_DATABASE, deletionEntry(MATCH_TABLE))); + assertPatternFilters(deleteDataNode(MATCH_DATABASE, deletionEntry(MISMATCH_TABLE))); + } + + private void assertTableDevicePlanMatches( + final BiFunction planFactory) { + assertPatternMatches( + planFactory.apply(MATCH_DATABASE, MATCH_TABLE), + planFactory.apply(MATCH_DATABASE, MISMATCH_TABLE), + planFactory.apply(MISMATCH_DATABASE, MATCH_TABLE)); + } + + private void assertPatternMatches(final PlanNode matchedInput, final PlanNode... filteredInputs) { Assert.assertEquals( - trueInput, + matchedInput, IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(trueInput, tablePattern) + .process(matchedInput, tablePattern) .orElseThrow(AssertionError::new)); + Arrays.stream(filteredInputs).forEach(this::assertPatternFilters); + } + + private void assertPatternFilters(final PlanNode filteredInput) { Assert.assertFalse( IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(falseInput1, tablePattern) - .isPresent()); - Assert.assertFalse( - IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process(falseInput2, tablePattern) + .process(filteredInput, tablePattern) .isPresent()); } - @Test - public void testDeleteData() { - Assert.assertEquals( - new RelationalDeleteDataNode( - new PlanNodeId(""), - new TableDeletionEntry( - new DeletionPredicate("ab"), new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)), - "db1"), - IoTDBSchemaRegionSource.TABLE_PATTERN_PARSE_VISITOR - .process( - new RelationalDeleteDataNode( - new PlanNodeId(""), - Arrays.asList( - new TableDeletionEntry( - new DeletionPredicate("ab"), - new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)), - new TableDeletionEntry( - new DeletionPredicate( - "ac", - new IDPredicate.And( - new IDPredicate.FullExactMatch( - DeviceIDFactory.getInstance() - .getDeviceID( - new PartialPath(new String[] {"ac", "device1"}))), - new IDPredicate.SegmentExactMatch("device2", 1))), - new TimeRange(0, 1))), - "db1"), - tablePattern) - .orElse(null)); + private CreateOrUpdateTableDeviceNode createOrUpdateTableDeviceNode( + final String database, final String table) { + return new CreateOrUpdateTableDeviceNode( + new PlanNodeId(""), + database, + table, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList()); + } + + private TableDeviceAttributeUpdateNode tableDeviceAttributeUpdateNode( + final String database, final String table) { + return new TableDeviceAttributeUpdateNode( + new PlanNodeId(""), database, table, null, null, null, null, null, null); + } + + private RelationalDeleteDataNode deleteDataNode( + final String database, final TableDeletionEntry... entries) { + return new RelationalDeleteDataNode(new PlanNodeId(""), Arrays.asList(entries), database); + } + + private TableDeletionEntry deletionEntry(final String table) { + return new TableDeletionEntry( + new DeletionPredicate(table), new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)); } } From 57a1cf33538341b0cb9137e4427a17ba2400e395 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:29:14 +0800 Subject: [PATCH 7/9] Improve pipe sink tests --- .../sink/PipeStatementEventSorterTest.java | 92 +++++------ .../sink/PipeTabletEventPlainBatchTest.java | 150 +++++++++--------- .../pipe/sink/PipeTabletEventSorterTest.java | 90 +++++------ 3 files changed, 155 insertions(+), 177 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementEventSorterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementEventSorterTest.java index 7c13d9764b3cf..d3d19fd9cd384 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementEventSorterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeStatementEventSorterTest.java @@ -233,81 +233,63 @@ public void testTableModelSort() throws Exception { } @Test - public void testTableModelSort1() throws Exception { - doTableModelTest1(); + public void testTableModelSortedDeduplicated() throws Exception { + doTableModelTest(false, false); } - public void doTableModelTest(final boolean hasDuplicates, final boolean isUnSorted) + @Test + public void testTableModelSortByTimestamp() throws Exception { + doTableModelSortByTimestampTest(); + } + + private static void doTableModelTest(final boolean hasDuplicates, final boolean isUnSorted) throws Exception { final Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, hasDuplicates, isUnSorted); - // Convert Tablet to Statement - InsertTabletStatement statement = new InsertTabletStatement(tablet, false, "test_db"); + final InsertTabletStatement statement = new InsertTabletStatement(tablet, false, "test_db"); - // Sort using Statement new PipeTableModelTabletEventSorter(statement).sortAndDeduplicateByDevIdTimestamp(); - - long[] timestamps = statement.getTimes(); - final Object[] columns = statement.getColumns(); - for (int i = 1; i < statement.getRowCount(); i++) { - long time = timestamps[i]; - Assert.assertTrue(time > timestamps[i - 1]); - Assert.assertEquals( - ((Binary[]) columns[0])[i], - new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(((long[]) columns[1])[i], (long) i); - Assert.assertEquals(((float[]) columns[2])[i], i * 1.0f, 0.001f); - Assert.assertEquals( - ((Binary[]) columns[3])[i], - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(((long[]) columns[4])[i], (long) i); - Assert.assertEquals(((int[]) columns[5])[i], i); - Assert.assertEquals(((double[]) columns[6])[i], i * 0.1, 0.0001); - // DATE is stored as int[] in Statement, not LocalDate[] - LocalDate expectedDate = PipeTabletEventSorterTest.getDate(i); - int expectedDateInt = - org.apache.tsfile.utils.DateUtils.parseDateExpressionToInt(expectedDate); - Assert.assertEquals(((int[]) columns[7])[i], expectedDateInt); - Assert.assertEquals( - ((Binary[]) columns[8])[i], - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - } + assertSortedTableModelStatement(statement); } - public void doTableModelTest1() throws Exception { + private static void doTableModelSortByTimestampTest() throws Exception { final Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, false, true); - // Convert Tablet to Statement - InsertTabletStatement statement = new InsertTabletStatement(tablet, false, "test_db"); + final InsertTabletStatement statement = new InsertTabletStatement(tablet, false, "test_db"); - // Sort using Statement new PipeTableModelTabletEventSorter(statement).sortByTimestampIfNecessary(); + assertSortedTableModelStatement(statement); + } - long[] timestamps = statement.getTimes(); + private static void assertSortedTableModelStatement(final InsertTabletStatement statement) + throws Exception { + final long[] timestamps = statement.getTimes(); final Object[] columns = statement.getColumns(); - for (int i = 1; i < statement.getRowCount(); i++) { - long time = timestamps[i]; - Assert.assertTrue(time > timestamps[i - 1]); + for (int i = 0; i < statement.getRowCount(); i++) { + final long time = timestamps[i]; + if (i > 0) { + Assert.assertTrue(time > timestamps[i - 1]); + } + Assert.assertEquals(i, time); Assert.assertEquals( - ((Binary[]) columns[0])[i], - new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(((long[]) columns[1])[i], (long) i); - Assert.assertEquals(((float[]) columns[2])[i], i * 1.0f, 0.001f); + new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8)), + ((Binary[]) columns[0])[i]); + Assert.assertEquals((long) i, ((long[]) columns[1])[i]); + Assert.assertEquals(i * 1.0f, ((float[]) columns[2])[i], 0.001f); Assert.assertEquals( - ((Binary[]) columns[3])[i], - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(((long[]) columns[4])[i], (long) i); - Assert.assertEquals(((int[]) columns[5])[i], i); - Assert.assertEquals(((double[]) columns[6])[i], i * 0.1, 0.0001); - // DATE is stored as int[] in Statement, not LocalDate[] - LocalDate expectedDate = PipeTabletEventSorterTest.getDate(i); - int expectedDateInt = + new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), + ((Binary[]) columns[3])[i]); + Assert.assertEquals((long) i, ((long[]) columns[4])[i]); + Assert.assertEquals(i, ((int[]) columns[5])[i]); + Assert.assertEquals(i * 0.1, ((double[]) columns[6])[i], 0.0001); + final LocalDate expectedDate = PipeTabletEventSorterTest.getDate(i); + final int expectedDateInt = org.apache.tsfile.utils.DateUtils.parseDateExpressionToInt(expectedDate); - Assert.assertEquals(((int[]) columns[7])[i], expectedDateInt); + Assert.assertEquals(expectedDateInt, ((int[]) columns[7])[i]); Assert.assertEquals( - ((Binary[]) columns[8])[i], - new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), + ((Binary[]) columns[8])[i]); } } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventPlainBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventPlainBatchTest.java index 1815c205b5774..fd1ee8369db27 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventPlainBatchTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventPlainBatchTest.java @@ -25,123 +25,125 @@ import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; +import java.util.Arrays; public class PipeTabletEventPlainBatchTest { @Test - public void constructTabletBatch() { - Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); - - Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); - - Assert.assertNotSame(tablet, copyTablet); - Assert.assertEquals(tablet.getRowSize(), copyTablet.getRowSize()); - - for (int i = 0; i < tablet.getSchemas().size(); i++) { - for (int j = 0; j < tablet.getRowSize(); j++) { - Assert.assertEquals(tablet.getValue(j, i), copyTablet.getValue(j, i)); - } - } + public void copyTabletPreservesSingleTablet() { + assertCopiedTablet(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); } @Test - public void constructTabletBatch1() { - Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); + public void copyTabletPreservesAppendedTablet() { + final Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); tablet.append(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); - Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); - - Assert.assertNotSame(tablet, copyTablet); - Assert.assertEquals(tablet.getRowSize(), copyTablet.getRowSize()); - for (int i = 0; i < tablet.getSchemas().size(); i++) { - for (int j = 0; j < tablet.getRowSize(); j++) { - Assert.assertEquals(tablet.getValue(j, i), copyTablet.getValue(j, i)); - } - } + assertCopiedTablet(tablet); } @Test - public void constructTabletBatch2() { + public void copyTabletPreservesNullValueColumn() { Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); tablet.append(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); - tablet = PipeTabletEventPlainBatch.copyTablet(tablet); tablet.getBitMaps()[1].markAll(); tablet.getValues()[1] = null; - Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); - - Assert.assertNotSame(tablet, copyTablet); - Assert.assertEquals(tablet.getRowSize(), copyTablet.getRowSize()); - - for (int i = 0; i < tablet.getSchemas().size(); i++) { - if (i == 1) { - Assert.assertTrue(tablet.getBitMaps()[i].isAllMarked()); - Assert.assertNull(copyTablet.getValues()[1]); - continue; - } - - for (int j = 0; j < tablet.getRowSize(); j++) { - Assert.assertEquals(tablet.getValue(j, i), copyTablet.getValue(j, i)); - } - } + final Tablet copyTablet = assertCopiedTablet(tablet); + Assert.assertTrue(tablet.getBitMaps()[1].isAllMarked()); + Assert.assertNull(copyTablet.getValues()[1]); } @Test - public void constructTabletBatch3() { + public void copyTabletPreservesAbsentBitmap() { Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); tablet.append(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); - tablet = PipeTabletEventPlainBatch.copyTablet(tablet); tablet.getBitMaps()[1] = null; - Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); + final Tablet copyTablet = assertCopiedTablet(tablet); + Assert.assertNull(tablet.getBitMaps()[1]); + Assert.assertNull(copyTablet.getBitMaps()[1]); + } - Assert.assertNotSame(tablet, copyTablet); - Assert.assertEquals(tablet.getRowSize(), copyTablet.getRowSize()); + @Test + public void copyTabletPreservesSparseNulls() { + final Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); + tablet.append(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); + final int[] rowIndices = new int[tablet.getSchemas().size()]; for (int i = 0; i < tablet.getSchemas().size(); i++) { - if (i == 1) { - Assert.assertNull(tablet.getBitMaps()[i]); - continue; - } - - for (int j = 0; j < tablet.getRowSize(); j++) { - Assert.assertEquals(tablet.getValue(j, i), copyTablet.getValue(j, i)); - } + rowIndices[i] = i % tablet.getRowSize(); + tablet.addValue(tablet.getSchemas().get(i).getMeasurementName(), rowIndices[i], null); } - } - @Test - public void constructTabletBatch4() { - Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, true, true); - tablet.append(PipeTabletEventSorterTest.generateTablet("test", 10, true, true)); + final Tablet copyTablet = assertCopiedTablet(tablet); - List rowIndices = new ArrayList<>(tablet.getSchemas().size()); - Random random = new Random(); for (int i = 0; i < tablet.getSchemas().size(); i++) { - int r = random.nextInt(tablet.getRowSize()); - rowIndices.add(r); - tablet.addValue(tablet.getSchemas().get(i).getMeasurementName(), r, null); + Assert.assertTrue(copyTablet.getBitMaps()[i].isMarked(rowIndices[i])); + Assert.assertNull(copyTablet.getValue(rowIndices[i], i)); } + } + + @Test + public void copyTabletIsIndependentFromSourceMutations() { + final Tablet tablet = PipeTabletEventSorterTest.generateTablet("test", 10, false, true); + final Tablet copyTablet = assertCopiedTablet(tablet); + + final int rowIndex = 1; + final long copiedTimestamp = copyTablet.getTimestamps()[rowIndex]; + final Object copiedValue = copyTablet.getValue(rowIndex, 1); + final boolean copiedMark = copyTablet.getBitMaps()[1].isMarked(rowIndex); + + tablet.getTimestamps()[rowIndex] = -1; + ((long[]) tablet.getValues()[1])[rowIndex] = -1; + tablet.getBitMaps()[1].mark(rowIndex); + + Assert.assertEquals(copiedTimestamp, copyTablet.getTimestamps()[rowIndex]); + Assert.assertEquals(copiedValue, copyTablet.getValue(rowIndex, 1)); + Assert.assertEquals(copiedMark, copyTablet.getBitMaps()[1].isMarked(rowIndex)); + } - Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); + private static Tablet assertCopiedTablet(final Tablet tablet) { + final Tablet copyTablet = PipeTabletEventPlainBatch.copyTablet(tablet); Assert.assertNotSame(tablet, copyTablet); + Assert.assertEquals(tablet.getTableName(), copyTablet.getTableName()); + Assert.assertNotSame(tablet.getSchemas(), copyTablet.getSchemas()); + Assert.assertEquals(tablet.getSchemas(), copyTablet.getSchemas()); + Assert.assertNotSame(tablet.getColumnTypes(), copyTablet.getColumnTypes()); + Assert.assertEquals(tablet.getColumnTypes(), copyTablet.getColumnTypes()); Assert.assertEquals(tablet.getRowSize(), copyTablet.getRowSize()); - + Assert.assertArrayEquals( + Arrays.copyOf(tablet.getTimestamps(), tablet.getRowSize()), + Arrays.copyOf(copyTablet.getTimestamps(), copyTablet.getRowSize())); + + if (tablet.getBitMaps() == null) { + Assert.assertNull(copyTablet.getBitMaps()); + } else { + Assert.assertNotSame(tablet.getBitMaps(), copyTablet.getBitMaps()); + } for (int i = 0; i < tablet.getSchemas().size(); i++) { + if (tablet.getBitMaps() != null && tablet.getBitMaps()[i] == null) { + Assert.assertNull(copyTablet.getBitMaps()[i]); + } else if (tablet.getBitMaps() != null) { + Assert.assertNotSame(tablet.getBitMaps()[i], copyTablet.getBitMaps()[i]); + } + + if (tablet.getValues()[i] == null) { + Assert.assertNull(copyTablet.getValues()[i]); + continue; + } + Assert.assertNotSame(tablet.getValues()[i], copyTablet.getValues()[i]); for (int j = 0; j < tablet.getRowSize(); j++) { - if (rowIndices.get(i) == j) { - Assert.assertTrue(tablet.getBitMaps()[i].isMarked(j)); - Assert.assertNull(tablet.getValue(j, i)); - continue; + if (tablet.getBitMaps() != null && tablet.getBitMaps()[i] != null) { + Assert.assertEquals( + tablet.getBitMaps()[i].isMarked(j), copyTablet.getBitMaps()[i].isMarked(j)); } Assert.assertEquals(tablet.getValue(j, i), copyTablet.getValue(j, i)); } } + return copyTablet; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventSorterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventSorterTest.java index eb5ffd475d361..b4adedccc1494 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventSorterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeTabletEventSorterTest.java @@ -88,10 +88,7 @@ public void testTreeModelDeduplicateAndSort() { Assert.assertEquals(indices.size(), tablet.getRowSize()); final long[] timestamps = Arrays.copyOfRange(tablet.getTimestamps(), 0, tablet.getRowSize()); - for (int i = 0; i < 3; ++i) { - Assert.assertArrayEquals( - timestamps, Arrays.copyOfRange((long[]) tablet.getValues()[0], 0, tablet.getRowSize())); - } + assertLongColumnsEqualTimestamps(tablet, timestamps); for (int i = 1; i < tablet.getRowSize(); ++i) { Assert.assertTrue(timestamps[i] > timestamps[i - 1]); @@ -179,10 +176,7 @@ public void testTreeModelSort() { Assert.assertFalse(tablet.isSorted()); long[] timestamps = Arrays.copyOfRange(tablet.getTimestamps(), 0, tablet.getRowSize()); - for (int i = 0; i < 3; ++i) { - Assert.assertArrayEquals( - timestamps, Arrays.copyOfRange((long[]) tablet.getValues()[0], 0, tablet.getRowSize())); - } + assertLongColumnsEqualTimestamps(tablet, timestamps); for (int i = 1; i < tablet.getRowSize(); ++i) { Assert.assertTrue(timestamps[i] != timestamps[i - 1]); @@ -198,10 +192,7 @@ public void testTreeModelSort() { Assert.assertEquals(indices.size(), tablet.getRowSize()); timestamps = Arrays.copyOfRange(tablet.getTimestamps(), 0, tablet.getRowSize()); - for (int i = 0; i < 3; ++i) { - Assert.assertArrayEquals( - timestamps, Arrays.copyOfRange((long[]) tablet.getValues()[0], 0, tablet.getRowSize())); - } + assertLongColumnsEqualTimestamps(tablet, timestamps); for (int i = 1; i < tablet.getRowSize(); ++i) { Assert.assertTrue(timestamps[i] > timestamps[i - 1]); @@ -227,53 +218,56 @@ public void testTableModelSort() { } @Test - public void testTableModelSort1() { - doTableModelTest1(); + public void testTableModelSortedDeduplicated() { + doTableModelTest(false, false); } - public void doTableModelTest(final boolean hasDuplicates, final boolean isUnSorted) { + @Test + public void testTableModelSortByTimestamp() { + doTableModelSortByTimestampTest(); + } + + private static void doTableModelTest(final boolean hasDuplicates, final boolean isUnSorted) { final Tablet tablet = generateTablet("test", 10, hasDuplicates, isUnSorted); new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp(); - long[] timestamps = tablet.getTimestamps(); - for (int i = 1; i < tablet.getRowSize(); i++) { - long time = timestamps[i]; - Assert.assertTrue(time > timestamps[i - 1]); - Assert.assertEquals( - tablet.getValue(i, 0), - new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(tablet.getValue(i, 1), (long) i); - Assert.assertEquals(tablet.getValue(i, 2), i * 1.0f); - Assert.assertEquals( - tablet.getValue(i, 3), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(tablet.getValue(i, 4), (long) i); - Assert.assertEquals(tablet.getValue(i, 5), i); - Assert.assertEquals(tablet.getValue(i, 6), i * 0.1); - Assert.assertEquals(tablet.getValue(i, 7), getDate(i)); - Assert.assertEquals( - tablet.getValue(i, 8), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - } + assertSortedTableModelTablet(tablet); } - public void doTableModelTest1() { + private static void doTableModelSortByTimestampTest() { final Tablet tablet = generateTablet("test", 10, false, true); new PipeTableModelTabletEventSorter(tablet).sortByTimestampIfNecessary(); - long[] timestamps = tablet.getTimestamps(); - for (int i = 1; i < tablet.getRowSize(); i++) { - long time = timestamps[i]; - Assert.assertTrue(time > timestamps[i - 1]); + assertSortedTableModelTablet(tablet); + } + + private static void assertLongColumnsEqualTimestamps( + final Tablet tablet, final long[] timestamps) { + for (int i = 0; i < tablet.getSchemas().size(); ++i) { + Assert.assertArrayEquals( + timestamps, Arrays.copyOfRange((long[]) tablet.getValues()[i], 0, tablet.getRowSize())); + } + } + + private static void assertSortedTableModelTablet(final Tablet tablet) { + final long[] timestamps = tablet.getTimestamps(); + for (int i = 0; i < tablet.getRowSize(); i++) { + final long time = timestamps[i]; + if (i > 0) { + Assert.assertTrue(time > timestamps[i - 1]); + } + Assert.assertEquals(i, time); Assert.assertEquals( - tablet.getValue(i, 0), - new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(tablet.getValue(i, 1), (long) i); - Assert.assertEquals(tablet.getValue(i, 2), i * 1.0f); + new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8)), + tablet.getValue(i, 0)); + Assert.assertEquals((long) i, tablet.getValue(i, 1)); + Assert.assertEquals(i * 1.0f, tablet.getValue(i, 2)); Assert.assertEquals( - tablet.getValue(i, 3), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); - Assert.assertEquals(tablet.getValue(i, 4), (long) i); - Assert.assertEquals(tablet.getValue(i, 5), i); - Assert.assertEquals(tablet.getValue(i, 6), i * 0.1); - Assert.assertEquals(tablet.getValue(i, 7), getDate(i)); + new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), tablet.getValue(i, 3)); + Assert.assertEquals((long) i, tablet.getValue(i, 4)); + Assert.assertEquals(i, tablet.getValue(i, 5)); + Assert.assertEquals(i * 0.1, tablet.getValue(i, 6)); + Assert.assertEquals(getDate(i), tablet.getValue(i, 7)); Assert.assertEquals( - tablet.getValue(i, 8), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8)), tablet.getValue(i, 8)); } } From 65880bb0ac461e29b70a4e277e31bf491b753073 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:50:07 +0800 Subject: [PATCH 8/9] Improve pipe unit test coverage --- .../parameter/PipeParametersTest.java | 52 +++ .../pipe/PipeRuntimeExceptionTest.java | 80 ++--- .../datastructure/PipeCommitQueueTest.java | 65 +++- .../PipeReceiverStatusHandlerTest.java | 331 ++++-------------- .../plugin/builtin/BuiltinPipePluginTest.java | 119 +++---- .../pipe/plugin/meta/PipePluginMetaTest.java | 44 ++- .../pipe/task/PipeSleepIntervalTest.java | 80 +++-- 7 files changed, 343 insertions(+), 428 deletions(-) diff --git a/iotdb-api/pipe-api/src/test/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParametersTest.java b/iotdb-api/pipe-api/src/test/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParametersTest.java index d16b8e2879d4a..3c068c4635fe8 100644 --- a/iotdb-api/pipe-api/src/test/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParametersTest.java +++ b/iotdb-api/pipe-api/src/test/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParametersTest.java @@ -22,7 +22,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Arrays; import java.util.HashMap; +import java.util.Map; public class PipeParametersTest { @@ -33,10 +35,60 @@ public void keyReducerTest() { Assert.assertEquals(true, parameters.getBoolean("with-quality")); Assert.assertEquals(true, parameters.getBoolean("opcua.with-quality")); + Assert.assertTrue(parameters.hasAnyAttributes("missing-key", "sink.opcua.with-quality")); // Invalid parameters.addAttribute("sink.source.opcua.value-name", "false"); parameters.addAttribute("opcua.sink.value-name", "false"); Assert.assertNull(parameters.getString("value-name")); } + + @Test + public void equivalentAttributeReplacementTest() { + final Map attributes = new HashMap<>(); + attributes.put("sink.opcua.with-quality", "false"); + final PipeParameters parameters = new PipeParameters(attributes); + + final Map replacements = new HashMap<>(); + replacements.put("connector.opcua.with-quality", "true"); + parameters.addOrReplaceEquivalentAttributes(new PipeParameters(replacements)); + + Assert.assertFalse(parameters.getAttribute().containsKey("sink.opcua.with-quality")); + Assert.assertEquals("true", parameters.getAttribute().get("connector.opcua.with-quality")); + } + + @Test + public void equivalentAttributeReplacementWithCloneTest() { + final Map attributes = new HashMap<>(); + attributes.put("sink.opcua.with-quality", "false"); + final PipeParameters parameters = new PipeParameters(attributes); + + final Map replacements = new HashMap<>(); + replacements.put("connector.opcua.with-quality", "true"); + final PipeParameters clonedParameters = + parameters.addOrReplaceEquivalentAttributesWithClone(new PipeParameters(replacements)); + + Assert.assertEquals("false", parameters.getAttribute().get("sink.opcua.with-quality")); + Assert.assertEquals("true", clonedParameters.getString("with-quality")); + Assert.assertEquals( + "true", + clonedParameters.getStringOrDefault( + Arrays.asList("missing-key", "sink.opcua.with-quality"), "default")); + } + + @Test + public void toStringShouldHideSensitiveValues() { + final Map attributes = new HashMap<>(); + attributes.put("sink.password", "secret"); + attributes.put("connector.ssl.trust-store-pwd", "credential"); + attributes.put("visible", "value"); + + final String parametersString = new PipeParameters(attributes).toString(); + + Assert.assertTrue(parametersString.contains("sink.password=******")); + Assert.assertTrue(parametersString.contains("connector.ssl.trust-store-pwd=******")); + Assert.assertTrue(parametersString.contains("visible=value")); + Assert.assertFalse(parametersString.contains("secret")); + Assert.assertFalse(parametersString.contains("credential")); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeExceptionTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeExceptionTest.java index b2d9fde0f0260..1afa92a9812f1 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeExceptionTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeExceptionTest.java @@ -24,60 +24,48 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; public class PipeRuntimeExceptionTest { @Test - public void testPipeRuntimeNonCriticalException() { - long currentTime = System.currentTimeMillis(); - PipeRuntimeNonCriticalException e = new PipeRuntimeNonCriticalException("test", currentTime); - Assert.assertEquals(new PipeRuntimeNonCriticalException("test", currentTime), e); - ByteBuffer buffer = ByteBuffer.allocate(32); - e.serialize(buffer); - buffer.position(0); - try { - PipeRuntimeNonCriticalException e1 = - (PipeRuntimeNonCriticalException) - PipeRuntimeExceptionType.deserializeFrom(PipeRuntimeMetaVersion.VERSION_2, buffer); - Assert.assertEquals(e.hashCode(), e1.hashCode()); - } catch (ClassCastException classCastException) { - Assert.fail(); + public void testPipeRuntimeExceptionSerde() throws IOException { + final long currentTime = System.currentTimeMillis(); + final List exceptions = + Arrays.asList( + new PipeRuntimeNonCriticalException("non-critical", currentTime), + new PipeRuntimeCriticalException("critical", currentTime), + new PipeRuntimeSinkCriticalException("sink-critical", currentTime), + new PipeRuntimeOutOfMemoryCriticalException("out-of-memory", currentTime)); + + for (final PipeRuntimeException exception : exceptions) { + assertSerde(exception); } } - @Test - public void testPipeRuntimeCriticalException() { - long currentTime = System.currentTimeMillis(); - PipeRuntimeCriticalException e = new PipeRuntimeCriticalException("test", currentTime); - Assert.assertEquals(new PipeRuntimeCriticalException("test", currentTime), e); - ByteBuffer buffer = ByteBuffer.allocate(32); - e.serialize(buffer); - buffer.position(0); - try { - PipeRuntimeCriticalException e1 = - (PipeRuntimeCriticalException) - PipeRuntimeExceptionType.deserializeFrom(PipeRuntimeMetaVersion.VERSION_2, buffer); - Assert.assertEquals(e.hashCode(), e1.hashCode()); - } catch (ClassCastException classCastException) { - Assert.fail(); - } + private static void assertSerde(final PipeRuntimeException exception) throws IOException { + Assert.assertEquals(exception, deserializeFromByteBuffer(exception)); + Assert.assertEquals(exception, deserializeFromInputStream(exception)); + Assert.assertEquals(exception.hashCode(), deserializeFromByteBuffer(exception).hashCode()); } - @Test - public void testPipeRuntimeConnectorCriticalException() { - long currentTime = System.currentTimeMillis(); - PipeRuntimeSinkCriticalException e = new PipeRuntimeSinkCriticalException("test", currentTime); - Assert.assertEquals(new PipeRuntimeSinkCriticalException("test", currentTime), e); - ByteBuffer buffer = ByteBuffer.allocate(32); - e.serialize(buffer); - buffer.position(0); - try { - PipeRuntimeSinkCriticalException e1 = - (PipeRuntimeSinkCriticalException) - PipeRuntimeExceptionType.deserializeFrom(PipeRuntimeMetaVersion.VERSION_2, buffer); - Assert.assertEquals(e.hashCode(), e1.hashCode()); - } catch (ClassCastException classCastException) { - Assert.fail(); - } + private static PipeRuntimeException deserializeFromByteBuffer( + final PipeRuntimeException exception) { + final ByteBuffer buffer = ByteBuffer.allocate(256); + exception.serialize(buffer); + buffer.flip(); + return PipeRuntimeExceptionType.deserializeFrom(PipeRuntimeMetaVersion.VERSION_2, buffer); + } + + private static PipeRuntimeException deserializeFromInputStream( + final PipeRuntimeException exception) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + exception.serialize(outputStream); + return PipeRuntimeExceptionType.deserializeFrom( + PipeRuntimeMetaVersion.VERSION_2, new ByteArrayInputStream(outputStream.toByteArray())); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java index 6a894b2104176..f6ac46f97a91d 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java @@ -42,27 +42,62 @@ public class PipeCommitQueueTest { @Test public void testCommitQueue() { final PipeCommitQueue pipeCommitQueue = new PipeCommitQueue(); - pipeCommitQueue.offer(new TestEnrichedEvent(1, new IoTProgressIndex(0, 1L))); - pipeCommitQueue.offer(new TestEnrichedEvent(3, new IoTProgressIndex(0, 3L))); + pipeCommitQueue.offer(event(1)); + pipeCommitQueue.offer(eventWithHook(3)); Assert.assertEquals(1, pipeCommitQueue.size()); - Assert.assertEquals(new IoTProgressIndex(0, 1L), pipeTaskMeta.getProgressIndex()); - TestEnrichedEvent nextEvent = new TestEnrichedEvent(5, new IoTProgressIndex(0, 5L)); - nextEvent.setShouldReportOnCommit(false); - pipeCommitQueue.offer(nextEvent); - pipeCommitQueue.offer(new TestEnrichedEvent(4, new IoTProgressIndex(0, 4L))); + assertProgressIndex(1); + + pipeCommitQueue.offer(eventWithHook(5, false)); + pipeCommitQueue.offer(eventWithHook(4)); Assert.assertEquals(1, pipeCommitQueue.size()); - nextEvent = new TestEnrichedEvent(2, new IoTProgressIndex(0, 2L)); - nextEvent.addOnCommittedHook(() -> commitHookTestSet.add(1)); - pipeCommitQueue.offer(nextEvent); + Assert.assertTrue(commitHookTestSet.isEmpty()); + + pipeCommitQueue.offer(eventWithHook(2)); Assert.assertEquals(0, pipeCommitQueue.size()); - Assert.assertEquals(new IoTProgressIndex(0, 4L), pipeTaskMeta.getProgressIndex()); - Assert.assertEquals(1, commitHookTestSet.size()); - // Test null progressIndex - pipeCommitQueue.offer(new TestEnrichedEvent(6, null)); + assertProgressIndex(4); + Assert.assertEquals(4, commitHookTestSet.size()); + Assert.assertTrue(commitHookTestSet.contains(2)); + Assert.assertTrue(commitHookTestSet.contains(3)); + Assert.assertTrue(commitHookTestSet.contains(4)); + Assert.assertTrue(commitHookTestSet.contains(5)); + + pipeCommitQueue.offer(eventWithHook(6, null)); + Assert.assertEquals(0, pipeCommitQueue.size()); + assertProgressIndex(4); + Assert.assertTrue(commitHookTestSet.contains(6)); + Assert.assertEquals(5, commitHookTestSet.size()); + } + + private TestEnrichedEvent event(final long commitId) { + return event(commitId, new IoTProgressIndex(0, commitId)); + } + + private TestEnrichedEvent eventWithHook(final long commitId) { + return eventWithHook(commitId, new IoTProgressIndex(0, commitId)); + } + + private TestEnrichedEvent eventWithHook(final long commitId, final boolean shouldReportOnCommit) { + final TestEnrichedEvent event = eventWithHook(commitId); + event.setShouldReportOnCommit(shouldReportOnCommit); + return event; + } + + private TestEnrichedEvent eventWithHook(final long commitId, final ProgressIndex progressIndex) { + final TestEnrichedEvent event = event(commitId, progressIndex); + event.addOnCommittedHook(() -> commitHookTestSet.add((int) commitId)); + return event; + } + + private TestEnrichedEvent event(final long commitId, final ProgressIndex progressIndex) { + return new TestEnrichedEvent(commitId, progressIndex); + } + + private void assertProgressIndex(final long searchIndex) { + Assert.assertEquals(new IoTProgressIndex(0, searchIndex), pipeTaskMeta.getProgressIndex()); } private class TestEnrichedEvent extends EnrichedEvent { - private ProgressIndex progressIndex; + private final ProgressIndex progressIndex; private TestEnrichedEvent(final long commitId, final ProgressIndex progressIndex) { this( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeReceiverStatusHandlerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeReceiverStatusHandlerTest.java index 38d2e015165dd..cd4352a4dfc04 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeReceiverStatusHandlerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeReceiverStatusHandlerTest.java @@ -23,11 +23,15 @@ import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.rpc.TSStatusCode; +import org.junit.After; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.slf4j.Marker; + +import java.util.Arrays; +import java.util.List; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE; @@ -36,267 +40,86 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE; public class PipeReceiverStatusHandlerTest { - @Test - public void testAuthLogger() { - final PipeReceiverStatusHandler handler = - new PipeReceiverStatusHandler( - CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE.equals("retry"), - CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE, - CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE, - CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE, - CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE, - true); - PipeReceiverStatusHandler.setLogger( - new Logger() { - @Override - public String getName() { - return null; - } - - @Override - public boolean isTraceEnabled() { - return false; - } - - @Override - public void trace(String msg) {} - - @Override - public void trace(String format, Object arg) {} - - @Override - public void trace(String format, Object arg1, Object arg2) {} - - @Override - public void trace(String format, Object... arguments) {} - - @Override - public void trace(String msg, Throwable t) {} - - @Override - public boolean isTraceEnabled(Marker marker) { - return false; - } - - @Override - public void trace(Marker marker, String msg) {} - - @Override - public void trace(Marker marker, String format, Object arg) {} - - @Override - public void trace(Marker marker, String format, Object arg1, Object arg2) {} - - @Override - public void trace(Marker marker, String format, Object... argArray) {} - - @Override - public void trace(Marker marker, String msg, Throwable t) {} - - @Override - public boolean isDebugEnabled() { - return false; - } - - @Override - public void debug(String msg) {} - - @Override - public void debug(String format, Object arg) {} - - @Override - public void debug(String format, Object arg1, Object arg2) {} - - @Override - public void debug(String format, Object... arguments) {} - - @Override - public void debug(String msg, Throwable t) {} - - @Override - public boolean isDebugEnabled(Marker marker) { - return false; - } - - @Override - public void debug(Marker marker, String msg) {} - - @Override - public void debug(Marker marker, String format, Object arg) {} - - @Override - public void debug(Marker marker, String format, Object arg1, Object arg2) {} - - @Override - public void debug(Marker marker, String format, Object... arguments) {} - - @Override - public void debug(Marker marker, String msg, Throwable t) {} - - @Override - public boolean isInfoEnabled() { - return false; - } - - @Override - public void info(String msg) {} - - @Override - public void info(String format, Object arg) {} - - @Override - public void info(String format, Object arg1, Object arg2) {} - - @Override - public void info(String format, Object... arguments) {} - - @Override - public void info(String msg, Throwable t) {} - - @Override - public boolean isInfoEnabled(Marker marker) { - return false; - } - - @Override - public void info(Marker marker, String msg) {} - - @Override - public void info(Marker marker, String format, Object arg) {} - - @Override - public void info(Marker marker, String format, Object arg1, Object arg2) {} - @Override - public void info(Marker marker, String format, Object... arguments) {} - - @Override - public void info(Marker marker, String msg, Throwable t) {} - - // Warn - @Override - public boolean isWarnEnabled() { - return true; - } - - @Override - public void warn(String msg) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(String format, Object arg) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(String format, Object... arguments) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(String format, Object arg1, Object arg2) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(String msg, Throwable t) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isWarnEnabled(Marker marker) { - return true; - } - - @Override - public void warn(Marker marker, String msg) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(Marker marker, String format, Object arg) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(Marker marker, String format, Object arg1, Object arg2) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(Marker marker, String format, Object... arguments) { - throw new UnsupportedOperationException(); - } - - @Override - public void warn(Marker marker, String msg, Throwable t) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isErrorEnabled() { - return false; - } - - @Override - public void error(String msg) {} - - @Override - public void error(String format, Object arg) {} - - @Override - public void error(String format, Object arg1, Object arg2) {} + @After + public void tearDown() { + PipeReceiverStatusHandler.setLogger(LoggerFactory.getLogger(PipeReceiverStatusHandler.class)); + } - @Override - public void error(String format, Object... arguments) {} + @Test + public void testAuthLogger() { + final Logger logger = Mockito.mock(Logger.class); + Mockito.when(logger.isWarnEnabled()).thenReturn(true); + PipeReceiverStatusHandler.setLogger(logger); - @Override - public void error(String msg, Throwable t) {} + final PipeReceiverStatusHandler handler = createHandler(true); + final String recordMessage = "root.sg.d1.s1=1"; - @Override - public boolean isErrorEnabled(Marker marker) { - return false; - } + handler.handle( + new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()), + "", + recordMessage); + handler.handle(new TSStatus(TSStatusCode.NO_PERMISSION.getStatusCode()), "", recordMessage); + Mockito.verify(logger, Mockito.never()) + .warn(Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any()); - @Override - public void error(Marker marker, String msg) {} + handler.handle( + new TSStatus(TSStatusCode.NO_PERMISSION.getStatusCode()), "", recordMessage, true); + handler.handle( + new TSStatus(TSStatusCode.METADATA_ERROR.getStatusCode()), + "No permissions for this operation, please add privilege WRITE_DATA", + recordMessage, + true); + + Mockito.verify(logger, Mockito.times(2)) + .warn( + Mockito.anyString(), + Mockito.eq("No permission"), + Mockito.eq(recordMessage), + Mockito.any(TSStatus.class)); + } - @Override - public void error(Marker marker, String format, Object arg) {} + @Test + public void testGetPriorStatusReturnsHighestKnownPriority() { + final List statusList = + Arrays.asList( + status(TSStatusCode.SUCCESS_STATUS), + status(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION), + status(TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION), + status(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION)); + + final TSStatus priorStatus = PipeReceiverStatusHandler.getPriorStatus(statusList); + + Assert.assertEquals( + TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode(), + priorStatus.getCode()); + Assert.assertEquals(statusList, priorStatus.getSubStatus()); + } - @Override - public void error(Marker marker, String format, Object arg1, Object arg2) {} + @Test + public void testGetPriorStatusReturnsFirstUnknownStatus() { + final TSStatus metadataError = status(TSStatusCode.METADATA_ERROR); + + Assert.assertSame( + metadataError, + PipeReceiverStatusHandler.getPriorStatus( + Arrays.asList( + status(TSStatusCode.SUCCESS_STATUS), + metadataError, + status(TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION)))); + } - @Override - public void error(Marker marker, String format, Object... arguments) {} + private static PipeReceiverStatusHandler createHandler(final boolean skipIfNoPrivileges) { + return new PipeReceiverStatusHandler( + CONNECTOR_EXCEPTION_CONFLICT_RESOLVE_STRATEGY_DEFAULT_VALUE.equals("retry"), + CONNECTOR_EXCEPTION_CONFLICT_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE, + CONNECTOR_EXCEPTION_CONFLICT_RECORD_IGNORED_DATA_DEFAULT_VALUE, + CONNECTOR_EXCEPTION_OTHERS_RETRY_MAX_TIME_SECONDS_DEFAULT_VALUE, + CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE, + skipIfNoPrivileges); + } - @Override - public void error(Marker marker, String msg, Throwable t) {} - }); - handler.handle( - new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()), - "", - ""); - handler.handle(new TSStatus(TSStatusCode.NO_PERMISSION.getStatusCode()), "", ""); - try { - handler.handle(new TSStatus(TSStatusCode.NO_PERMISSION.getStatusCode()), "", "", true); - Assert.fail(); - } catch (final UnsupportedOperationException e) { - // Expected - } - handler.handle(new TSStatus(TSStatusCode.NO_PERMISSION.getStatusCode()), "", ""); - try { - handler.handle( - new TSStatus(TSStatusCode.METADATA_ERROR.getStatusCode()) - .setMessage("No permissions for this operation, please add privilege WRITE_DATA"), - "", - "", - true); - Assert.fail(); - } catch (final UnsupportedOperationException e) { - // Expected - } - PipeReceiverStatusHandler.setLogger(LoggerFactory.getLogger(PipeReceiverStatusHandler.class)); + private static TSStatus status(final TSStatusCode statusCode) { + return new TSStatus(statusCode.getStatusCode()); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/builtin/BuiltinPipePluginTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/builtin/BuiltinPipePluginTest.java index 95f549b5e6cca..9ce3b58bd0c51 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/builtin/BuiltinPipePluginTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/builtin/BuiltinPipePluginTest.java @@ -41,37 +41,12 @@ import org.junit.Test; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class BuiltinPipePluginTest { @Test public void testBuildInPipePlugin() { - PipeExtractor extractor = new IoTDBSource(); - try { - extractor.validate(mock(PipeParameterValidator.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - extractor.customize( - mock(PipeParameters.class), mock(PipeExtractorRuntimeConfiguration.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - extractor.start(); - Assert.fail(); - } catch (Exception ignored) { - } - try { - extractor.supply(); - Assert.fail(); - } catch (Exception ignored) { - } - try { - extractor.close(); - Assert.fail(); - } catch (Exception ignored) { - } + testExtractorAllThrow(new IoTDBSource()); PipeProcessor processor = new DoNothingProcessor(); try { @@ -85,18 +60,26 @@ public void testBuildInPipePlugin() { } catch (Exception ignored) { Assert.fail(); } + + final EventCollector collector = mock(EventCollector.class); + final TabletInsertionEvent tabletInsertionEvent = mock(TabletInsertionEvent.class); + final TsFileInsertionEvent tsFileInsertionEvent = mock(TsFileInsertionEvent.class); + final Event event = mock(Event.class); try { - processor.process(mock(TabletInsertionEvent.class), mock(EventCollector.class)); + processor.process(tabletInsertionEvent, collector); + verify(collector).collect(tabletInsertionEvent); } catch (Exception ignored) { Assert.fail(); } try { - processor.process(mock(TsFileInsertionEvent.class), mock(EventCollector.class)); + processor.process(tsFileInsertionEvent, collector); + verify(collector).collect(tsFileInsertionEvent); } catch (Exception ignored) { Assert.fail(); } try { - processor.process(mock(Event.class), mock(EventCollector.class)); + processor.process(event, collector); + verify(collector).collect(event); } catch (Exception ignored) { Assert.fail(); } @@ -125,47 +108,39 @@ public void testBuildInPipePlugin() { testConnectorAllThrow(new IoTDBThriftSink()); } - private void testConnectorAllThrow(PipeConnector connector) { - try { - connector.validate(mock(PipeParameterValidator.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.customize( - mock(PipeParameters.class), mock(PipeConnectorRuntimeConfiguration.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.handshake(); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.heartbeat(); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.transfer(mock(TabletInsertionEvent.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.transfer(mock(TsFileInsertionEvent.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.transfer(mock(Event.class)); - Assert.fail(); - } catch (Exception ignored) { - } - try { - connector.close(); - Assert.fail(); - } catch (Exception ignored) { - } + private void testExtractorAllThrow(final PipeExtractor extractor) { + Assert.assertThrows( + UnsupportedOperationException.class, + () -> extractor.validate(mock(PipeParameterValidator.class))); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + extractor.customize( + mock(PipeParameters.class), mock(PipeExtractorRuntimeConfiguration.class))); + Assert.assertThrows(UnsupportedOperationException.class, extractor::start); + Assert.assertThrows(UnsupportedOperationException.class, extractor::supply); + Assert.assertThrows(UnsupportedOperationException.class, extractor::close); + } + + private void testConnectorAllThrow(final PipeConnector connector) { + Assert.assertThrows( + UnsupportedOperationException.class, + () -> connector.validate(mock(PipeParameterValidator.class))); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> + connector.customize( + mock(PipeParameters.class), mock(PipeConnectorRuntimeConfiguration.class))); + Assert.assertThrows(UnsupportedOperationException.class, connector::handshake); + Assert.assertThrows(UnsupportedOperationException.class, connector::heartbeat); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> connector.transfer(mock(TabletInsertionEvent.class))); + Assert.assertThrows( + UnsupportedOperationException.class, + () -> connector.transfer(mock(TsFileInsertionEvent.class))); + Assert.assertThrows( + UnsupportedOperationException.class, () -> connector.transfer(mock(Event.class))); + Assert.assertThrows(UnsupportedOperationException.class, connector::close); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/meta/PipePluginMetaTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/meta/PipePluginMetaTest.java index b5b854adc4de4..aeed137ea2440 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/meta/PipePluginMetaTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/plugin/meta/PipePluginMetaTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.plugin.meta.ConfigNodePipePluginMetaKeeper; import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper; +import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.junit.Assert; import org.junit.Test; @@ -33,29 +34,48 @@ public class PipePluginMetaTest { @Test - public void testConfigNodePipePluginMetaKeeper() { + public void testConfigNodePipePluginMetaKeeper() throws IOException { ConfigNodePipePluginMetaKeeper keeper = new ConfigNodePipePluginMetaKeeper(); Assert.assertFalse(keeper.containsJar("test.jar")); keeper.addJarNameAndMd5("test.jar", "md5"); + keeper.addJarNameAndMd5("test.jar", "md5"); Assert.assertTrue(keeper.jarNameExistsAndMatchesMd5("test.jar", "md5")); + Assert.assertFalse(keeper.jarNameExistsAndMatchesMd5("test.jar", "wrong-md5")); + + final PipePluginMeta pluginMeta = + new PipePluginMeta( + "externalPlugin", "org.example.ExternalPlugin", false, "test.jar", "md5"); + keeper.addPipePluginMeta(pluginMeta.getPluginName(), pluginMeta); + Assert.assertEquals("EXTERNALPLUGIN", keeper.getPluginNameByJarName("test.jar")); + + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + keeper.processTakeSnapshot(outputStream); + + keeper.removeJarNameAndMd5IfPossible("test.jar"); + Assert.assertTrue(keeper.containsJar("test.jar")); + keeper.removeJarNameAndMd5IfPossible("test.jar"); + keeper.removePipePluginMeta(pluginMeta.getPluginName()); + Assert.assertFalse(keeper.containsJar("test.jar")); + Assert.assertFalse(keeper.containsPipePlugin(pluginMeta.getPluginName())); - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - keeper.processTakeSnapshot(outputStream); - keeper.removeJarNameAndMd5IfPossible("test.jar"); - ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); - keeper.processLoadSnapshot(inputStream); - Assert.assertTrue(keeper.jarNameExistsAndMatchesMd5("test.jar", "md5")); - } catch (IOException e) { - Assert.fail(); - } + keeper.processLoadSnapshot(new ByteArrayInputStream(outputStream.toByteArray())); + + Assert.assertTrue(keeper.jarNameExistsAndMatchesMd5("test.jar", "md5")); + Assert.assertEquals(pluginMeta, keeper.getPipePluginMeta(pluginMeta.getPluginName())); + Assert.assertEquals("EXTERNALPLUGIN", keeper.getPluginNameByJarName("test.jar")); } @Test public void testDataNodePipePluginMetaKeeper() { - DataNodePipePluginMetaKeeper keeper = new DataNodePipePluginMetaKeeper(); + final DataNodePipePluginMetaKeeper keeper = new DataNodePipePluginMetaKeeper(); Assert.assertEquals( BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginClass(), keeper.getBuiltinPluginClass(BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName())); + Assert.assertEquals( + BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginClass(), + keeper.getBuiltinPluginClass( + BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName().toUpperCase())); + Assert.assertTrue( + keeper.containsPipePlugin(BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName())); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeSleepIntervalTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeSleepIntervalTest.java index 07ae50e992e6b..bb7f487d8ab39 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeSleepIntervalTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeSleepIntervalTest.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException; import org.apache.iotdb.commons.pipe.agent.task.subtask.PipeAbstractSinkSubtask; -import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.junit.After; @@ -31,7 +30,13 @@ import org.junit.Before; import org.junit.Test; +import java.util.concurrent.TimeUnit; + public class PipeSleepIntervalTest { + private static final long INIT_SLEEP_INTERVAL_MS = 25L; + private static final long MAX_SLEEP_INTERVAL_MS = 50L; + private static final long SLEEP_ASSERTION_TOLERANCE_MS = 5L; + private long oldPipeSinkSubtaskSleepIntervalInitMs; private long oldPipeSinkSubtaskSleepIntervalMaxMs; @@ -40,8 +45,8 @@ public void setUp() throws Exception { final CommonConfig config = CommonDescriptor.getInstance().getConfig(); oldPipeSinkSubtaskSleepIntervalInitMs = config.getPipeSinkSubtaskSleepIntervalInitMs(); oldPipeSinkSubtaskSleepIntervalMaxMs = config.getPipeSinkSubtaskSleepIntervalMaxMs(); - config.setPipeSinkSubtaskSleepIntervalInitMs(25L); - config.setPipeSinkSubtaskSleepIntervalMaxMs(50L); + config.setPipeSinkSubtaskSleepIntervalInitMs(INIT_SLEEP_INTERVAL_MS); + config.setPipeSinkSubtaskSleepIntervalMaxMs(MAX_SLEEP_INTERVAL_MS); } @After @@ -52,32 +57,49 @@ public void tearDown() throws Exception { } @Test - public void test() { - try (final PipeAbstractSinkSubtask subtask = - new PipeAbstractSinkSubtask(null, 0, null) { - @Override - protected String getRootCause(Throwable throwable) { - return null; - } - - @Override - protected void report(EnrichedEvent event, PipeRuntimeException exception) {} - - @Override - protected boolean executeOnce() { - return false; - } - }) { - long startTime = System.currentTimeMillis(); - subtask.sleep4NonReportException(); - Assert.assertTrue( - System.currentTimeMillis() - startTime - >= PipeConfig.getInstance().getPipeSinkSubtaskSleepIntervalInitMs()); - startTime = System.currentTimeMillis() - startTime; - subtask.sleep4NonReportException(); - Assert.assertTrue( - System.currentTimeMillis() - startTime - >= PipeConfig.getInstance().getPipeSinkSubtaskSleepIntervalInitMs()); + public void testSleepIntervalStopsIncreasingAtMax() { + try (final TestSinkSubtask subtask = new TestSinkSubtask()) { + Assert.assertEquals(INIT_SLEEP_INTERVAL_MS, subtask.getSleepInterval()); + + assertSleepAtLeast(subtask, MAX_SLEEP_INTERVAL_MS); + Assert.assertEquals(MAX_SLEEP_INTERVAL_MS, subtask.getSleepInterval()); + + assertSleepAtLeast(subtask, MAX_SLEEP_INTERVAL_MS); + Assert.assertEquals(MAX_SLEEP_INTERVAL_MS, subtask.getSleepInterval()); + } + } + + private static void assertSleepAtLeast( + final PipeAbstractSinkSubtask subtask, final long expectedSleepMs) { + final long startTime = System.nanoTime(); + subtask.sleep4NonReportException(); + + Assert.assertTrue( + System.nanoTime() - startTime + >= TimeUnit.MILLISECONDS.toNanos(expectedSleepMs - SLEEP_ASSERTION_TOLERANCE_MS)); + } + + private static class TestSinkSubtask extends PipeAbstractSinkSubtask { + + private TestSinkSubtask() { + super(null, 0, null); + } + + private long getSleepInterval() { + return sleepInterval; + } + + @Override + protected String getRootCause(final Throwable throwable) { + return null; + } + + @Override + protected void report(final EnrichedEvent event, final PipeRuntimeException exception) {} + + @Override + protected boolean executeOnce() { + return false; } } } From 7cde6a74da704950f4b52e51b9e8d35a3393c6d4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:37:11 +0800 Subject: [PATCH 9/9] Update PipeCommitQueueTest.java --- .../datastructure/PipeCommitQueueTest.java | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java index f6ac46f97a91d..aa18acc814333 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/PipeCommitQueueTest.java @@ -100,7 +100,7 @@ private class TestEnrichedEvent extends EnrichedEvent { private final ProgressIndex progressIndex; private TestEnrichedEvent(final long commitId, final ProgressIndex progressIndex) { - this( + super( null, 0, PipeCommitQueueTest.this.pipeTaskMeta, @@ -123,32 +123,6 @@ public ProgressIndex getProgressIndex() { /////////////////////////////// Useless logic /////////////////////////////// - protected TestEnrichedEvent( - final String pipeName, - final long creationTime, - final PipeTaskMeta pipeTaskMeta, - final TreePattern treePattern, - final TablePattern tablePattern, - final String userId, - final String userName, - final String cliHostname, - final boolean skipIfNoPrivileges, - final long startTime, - final long endTime) { - super( - pipeName, - creationTime, - pipeTaskMeta, - treePattern, - tablePattern, - userId, - userName, - cliHostname, - skipIfNoPrivileges, - startTime, - endTime); - } - @Override public boolean internallyIncreaseResourceReferenceCount(String holderMessage) { return false;