diff --git a/query/common/CardinalityEstimation.cpp b/query/common/CardinalityEstimation.cpp index 1b0848659b..ce1ac27e22 100644 --- a/query/common/CardinalityEstimation.cpp +++ b/query/common/CardinalityEstimation.cpp @@ -25,6 +25,18 @@ size_t CardinalityEstimation::estimateEdgeCount() const { return reader.getEdgeCount(); } +size_t CardinalityEstimation::estimateOutEdgeCount(const LabelSet& labelset) const { + GraphReader reader(_graphView); + + return reader.getOutEdgeCountMatchingLabelset(labelset.handle()); +} + +size_t CardinalityEstimation::estimateInEdgeCount(const LabelSet& labelset) const { + GraphReader reader(_graphView); + + return reader.getInEdgeCountMatchingLabelset(labelset.handle()); +} + bool CardinalityEstimation::shouldPreferCartesian(const LabelSet& left, const LabelSet& right, size_t queryLimit) const { const size_t leftCount = estimateNodeCount(left); const size_t rightCount = estimateNodeCount(right); diff --git a/query/common/CardinalityEstimation.h b/query/common/CardinalityEstimation.h index 03d378117f..c979a210b6 100644 --- a/query/common/CardinalityEstimation.h +++ b/query/common/CardinalityEstimation.h @@ -26,6 +26,11 @@ class CardinalityEstimation { // count is what a by-type scan is read at, an upper bound on its rows. size_t estimateEdgeCount() const; + // The edges a scan of the out-edge (in-edge) index walks under a label set: those of the + // nodes matching it, deleted ones included. + size_t estimateOutEdgeCount(const LabelSet& labelset) const; + size_t estimateInEdgeCount(const LabelSet& labelset) const; + // Returns true when cartesian product + filter is likely cheaper // than value hash join, either because the product is small or // because a small LIMIT makes early-termination effective. diff --git a/query/ir/codegen/DBPassPipeline.cpp b/query/ir/codegen/DBPassPipeline.cpp index 85e7d92e27..1b34d2c30c 100644 --- a/query/ir/codegen/DBPassPipeline.cpp +++ b/query/ir/codegen/DBPassPipeline.cpp @@ -36,7 +36,7 @@ const std::array dbPassPipeline = { [](const mlir::db::DBPassContext*) { return mlir::db::createNarrowEdgeTypeReads(); }, [](const mlir::db::DBPassContext*) { return mlir::db::createFuseScanOutEdgesByLabel(); }, [](const mlir::db::DBPassContext*) { return mlir::db::createFuseScanInEdgesByLabel(); }, - [](const mlir::db::DBPassContext*) { return mlir::db::createFuseScanEdgesByEndpointLabel(); }, + [](const mlir::db::DBPassContext* context) { return mlir::db::createFuseScanEdgesByEndpointLabel(context); }, [](const mlir::db::DBPassContext*) { return mlir::db::createFuseEdgesByEndpointLabel(); }, [](const mlir::db::DBPassContext*) { return mlir::db::createRemoveRedundantLabelChecks(); }, [](const mlir::db::DBPassContext*) { return mlir::db::createFuseExploreEndConstraint(); }, diff --git a/query/ir/dialect/db/DBOps.cpp b/query/ir/dialect/db/DBOps.cpp index 35ad0f5031..fab53f308e 100644 --- a/query/ir/dialect/db/DBOps.cpp +++ b/query/ir/dialect/db/DBOps.cpp @@ -595,6 +595,18 @@ void ScanInEdgesByLabelSrc::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { } } +void ScanOutEdgesByLabelSrcTgt::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { + for (Value result : getResults()) { + setNameFn(result, ""); + } +} + +void ScanInEdgesByLabelSrcTgt::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { + for (Value result : getResults()) { + setNameFn(result, ""); + } +} + // Ensures each variable has a numeric name void GetOutEdges::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { for (Value result : getResults()) { @@ -1360,6 +1372,22 @@ LogicalResult ScanInEdgesByLabelSrc::verify() { return success(); } +LogicalResult ScanOutEdgesByLabelSrcTgt::verify() { + if (getSrcLabels().empty() || getTgtLabels().empty()) { + return emitOpError("requires at least one label on each end"); + } + + return success(); +} + +LogicalResult ScanInEdgesByLabelSrcTgt::verify() { + if (getSrcLabels().empty() || getTgtLabels().empty()) { + return emitOpError("requires at least one label on each end"); + } + + return success(); +} + LogicalResult GetOutEdgesByLabel::verify() { if (getLabels().empty()) { return emitOpError("requires at least one label"); diff --git a/query/ir/dialect/db/DBOps.td b/query/ir/dialect/db/DBOps.td index cc6303beb5..4e1eed87d0 100644 --- a/query/ir/dialect/db/DBOps.td +++ b/query/ir/dialect/db/DBOps.td @@ -1079,6 +1079,80 @@ def ScanInEdgesByLabelSrc : TuringOp<"scan_in_edges_by_label_src", [Pure, Declar }]; } +/** +* ScanOutEdgesByLabelSrcTgt +*/ +def ScanOutEdgesByLabelSrcTgt : TuringOp<"scan_out_edges_by_label_src_tgt", [Pure, DeclareOpInterfaceMethods]> { + let summary = "Produce every edge whose source and target each carry every given label of their end as four row-aligned columns"; + + let description = [{ + The both-ends sibling of db.scan_out_edges_by_label_src: it produces the edges + whose source's label set is a superset of `src_labels` and whose target's is a + superset of `tgt_labels`, in the same four row-aligned columns: + + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src_tgt(["Person"], ["Interest"]) : !db.column, !db.column, !db.column, !db.column + + It is `MATCH (a:Person)-->(b:Interest)`: the edges db.scan_out_edges_by_label_src + keeps once a db.check_label_constraint over its target column has dropped the + rest. It reads the index keyed by the source's label set, as that scan does, and + checks each edge's target against `tgt_labels`. db.scan_in_edges_by_label_src_tgt + produces the same rows off the index keyed by the target's label set. + + Neither label list may be empty: a scan labelled on one end only is + db.scan_out_edges_by_label_src or db.scan_in_edges_by_label_tgt. + }]; + + let arguments = (ins LabelRefArrayAttr:$src_labels, LabelRefArrayAttr:$tgt_labels); + + let results = (outs ColumnNodeIDs:$srcids, + ColumnEdgeIDs:$eids, + ColumnEdgeTypes:$etypes, + ColumnNodeIDs:$tgtids); + let hasVerifier = 1; + + let assemblyFormat = [{ + `(` $src_labels `,` $tgt_labels `)` attr-dict `:` qualified(type($srcids)) `,` qualified(type($eids)) `,` + qualified(type($etypes)) `,` qualified(type($tgtids)) + }]; +} + +/** +* ScanInEdgesByLabelSrcTgt +*/ +def ScanInEdgesByLabelSrcTgt : TuringOp<"scan_in_edges_by_label_src_tgt", [Pure, DeclareOpInterfaceMethods]> { + let summary = "Produce every edge whose target and source each carry every given label of their end, read off the target's index"; + + let description = [{ + The both-ends sibling of db.scan_in_edges_by_label_tgt: it produces the edges + whose source's label set is a superset of `src_labels` and whose target's is a + superset of `tgt_labels`: + + %srcs, %eids, %etypes, %tgts = db.scan_in_edges_by_label_src_tgt(["Interest"], ["Person"]) : !db.column, !db.column, !db.column, !db.column + + It is `MATCH (a:Person)<--(b:Interest)`: the edges db.scan_in_edges_by_label_tgt + keeps once a db.check_label_constraint over its source column has dropped the + rest. It reads the index keyed by the target's label set, as that scan does, and + checks each edge's source against `src_labels`. The rows are those of + db.scan_out_edges_by_label_src_tgt; the two differ in the index they read. + + Neither label list may be empty: a scan labelled on one end only is + db.scan_in_edges_by_label_tgt or db.scan_out_edges_by_label_src. + }]; + + let arguments = (ins LabelRefArrayAttr:$src_labels, LabelRefArrayAttr:$tgt_labels); + + let results = (outs ColumnNodeIDs:$srcids, + ColumnEdgeIDs:$eids, + ColumnEdgeTypes:$etypes, + ColumnNodeIDs:$tgtids); + let hasVerifier = 1; + + let assemblyFormat = [{ + `(` $src_labels `,` $tgt_labels `)` attr-dict `:` qualified(type($srcids)) `,` qualified(type($eids)) `,` + qualified(type($etypes)) `,` qualified(type($tgtids)) + }]; +} + /** * GetOutEdges */ diff --git a/query/ir/dialect/db/DBPasses.cpp b/query/ir/dialect/db/DBPasses.cpp index 3b8c1dc813..ce83faa60d 100644 --- a/query/ir/dialect/db/DBPasses.cpp +++ b/query/ir/dialect/db/DBPasses.cpp @@ -3563,16 +3563,119 @@ struct FuseScanInEdgesByLabel : public impl::FuseScanInEdgesByLabelBase(op)) { + return true; + } else if (isa(op)) { + scanLabels._source = op->getAttrOfType("labels"); + return true; + } else if (isa(op)) { + scanLabels._target = op->getAttrOfType("labels"); + scanLabels._indexedByTarget = true; + return true; + } else if (ScanOutEdgesByLabelSrcTgt scan = dyn_cast(op)) { + scanLabels._source = scan.getSrcLabels(); + scanLabels._target = scan.getTgtLabels(); + return true; + } else if (ScanInEdgesByLabelSrcTgt scan = dyn_cast(op)) { + scanLabels._source = scan.getSrcLabels(); + scanLabels._target = scan.getTgtLabels(); + scanLabels._indexedByTarget = true; + return true; + } + + return false; +} + +// The label set a scan's label names stand for, resolved against the schema, and whether +// the schema holds every one of them. +bool collectScanLabels(ArrayAttr labelNames, const ::db::GraphMetadata& metadata, ::db::LabelSet& labels) { + const ::db::LabelMap& labelMap = metadata.labels(); + bool resolvesEveryLabel = true; + for (const Attribute labelAttr : labelNames) { + const StringAttr nameAttr = dyn_cast(labelAttr); + if (!nameAttr) { + continue; + } + + const llvm::StringRef name = nameAttr.getValue(); + const std::optional<::db::LabelID> label = labelMap.get(std::string_view(name.data(), name.size())); + + if (label) { + labels.set(*label); + } else { + resolvesEveryLabel = false; + } + } + + return resolvesEveryLabel; +} + +// The nodes carrying every one of these labels, none when the schema lacks one of them. +size_t estimateLabelledNodes(ArrayAttr labelNames, + const ::db::GraphMetadata& metadata, + const ::db::CardinalityEstimation& estimation) { + ::db::LabelSet labels; + if (!collectScanLabels(labelNames, metadata, labels)) { + return 0; + } + + return estimation.estimateNodeCount(labels); +} + +// The edges the index keyed by one end's labels holds under these labels, which is what a +// scan of it walks, and none when the schema lacks one of them. +size_t estimateIndexedEdges(ArrayAttr labelNames, + bool indexedByTarget, + const ::db::GraphMetadata& metadata, + const ::db::CardinalityEstimation& estimation) { + ::db::LabelSet labels; + if (!collectScanLabels(labelNames, metadata, labels)) { + return 0; + } else if (indexedByTarget) { + return estimation.estimateInEdgeCount(labels); + } + + return estimation.estimateOutEdgeCount(labels); +} + +// The index a scan labelled on both ends walks: the one holding fewer edges under its end's +// labels, the source's on a tie. With no graph to count, the one the scan already walks. +bool indexesByTarget(const EdgeScanLabels& scanLabels, const DBPassContext& context) { + if (!context._view) { + return scanLabels._indexedByTarget; + } + + const ::db::GraphView& view = *context._view; + const ::db::GraphMetadata& metadata = view.metadata(); + const ::db::CardinalityEstimation estimation(view); + + const size_t sourceEdges = estimateIndexedEdges(scanLabels._source, false, metadata, estimation); + const size_t targetEdges = estimateIndexedEdges(scanLabels._target, true, metadata, estimation); + + return targetEdges < sourceEdges; +} + // An edge scan whose rows are then cut down to the edges one endpoint of which carries a -// set of labels: the by-label edge scan spelled the long way, since the index keyed by that -// endpoint's label set holds exactly those edges and never builds the rows the filter goes -// on to drop. +// set of labels: the scan reading that endpoint by those labels as well, which never builds +// the rows the filter goes on to drop. struct EndpointLabelledEdgeScan { - ScanEdges _scan; + FilterOp _filter; + Operation* _scan {nullptr}; GetNodeLabelSet _labelSet; CheckLabelConstraint _check; - ArrayAttr _labels; + EdgeScanLabels _scanLabels; + llvm::SmallVector _endLabels; bool _labelledTarget {false}; + bool _endWasLabelled {false}; }; bool matchEndpointLabelledEdgeScan(FilterOp filter, EndpointLabelledEdgeScan& labelledScan) { @@ -3581,8 +3684,8 @@ bool matchEndpointLabelledEdgeScan(FilterOp filter, EndpointLabelledEdgeScan& la return false; } - const ArrayAttr labels = check.getConjunction(); - if (!labels) { + const ArrayAttr checkLabels = check.getConjunction(); + if (!checkLabels) { return false; } @@ -3592,19 +3695,22 @@ bool matchEndpointLabelledEdgeScan(FilterOp filter, EndpointLabelledEdgeScan& la } const Value labelledColumn = labelSet.getInputNodes(); - ScanEdges scan = labelledColumn.getDefiningOp(); - if (!scan) { + Operation* const scan = labelledColumn.getDefiningOp(); + EdgeScanLabels scanLabels; + if (!scan || !readEdgeScanLabels(scan, scanLabels)) { return false; } - const bool labelledTarget = labelledColumn == scan.getTgtids(); - const bool labelledSource = labelledColumn == scan.getSrcids(); + constexpr size_t SRC_RESULT_INDEX = 0; + constexpr size_t TGT_RESULT_INDEX = 3; + const bool labelledTarget = labelledColumn == scan->getResult(TGT_RESULT_INDEX); + const bool labelledSource = labelledColumn == scan->getResult(SRC_RESULT_INDEX); if (!labelledTarget && !labelledSource) { return false; } for (const Value column : filter.getColumnsToFilter()) { - if (column.getDefiningOp() != scan.getOperation()) { + if (column.getDefiningOp() != scan) { return false; } } @@ -3625,66 +3731,99 @@ bool matchEndpointLabelledEdgeScan(FilterOp filter, EndpointLabelledEdgeScan& la return false; } - labelledScan = EndpointLabelledEdgeScan {._scan = scan, - ._labelSet = labelSet, - ._check = check, - ._labels = labels, - ._labelledTarget = labelledTarget}; + const ArrayAttr endLabels = labelledTarget ? scanLabels._target : scanLabels._source; + if (endLabels) { + labelledScan._endLabels.assign(endLabels.begin(), endLabels.end()); + } + + for (const Attribute label : checkLabels) { + if (!llvm::is_contained(labelledScan._endLabels, label)) { + labelledScan._endLabels.push_back(label); + } + } + + labelledScan._filter = filter; + labelledScan._scan = scan; + labelledScan._labelSet = labelSet; + labelledScan._check = check; + labelledScan._scanLabels = scanLabels; + labelledScan._labelledTarget = labelledTarget; + labelledScan._endWasLabelled = endLabels != nullptr; return true; } -void fuseScanEdgesByEndpointLabel(FilterOp filter, - const EndpointLabelledEdgeScan& labelledScan, - mlir::OpBuilder& builder) { - ScanEdges scan = labelledScan._scan; - Operation* const scanOp = scan.getOperation(); +// The by-label edge scan reading the endpoints by these labels. A scan labelled on one end +// walks that end's index; one labelled on both walks the index _indexedByTarget names. +Operation* createLabelledEdgeScan(Operation* scan, const EdgeScanLabels& scanLabels, mlir::RewriterBase& rewriter) { + const Location loc = scan->getLoc(); + const mlir::TypeRange types = scan->getResultTypes(); - builder.setInsertionPoint(scanOp); + if (!scanLabels._source) { + return rewriter.create(loc, types, scanLabels._target); + } else if (!scanLabels._target) { + return rewriter.create(loc, types, scanLabels._source); + } else if (scanLabels._indexedByTarget) { + return rewriter.create(loc, types, scanLabels._source, scanLabels._target); + } - // Both by-label scans declare the same four results in the same order as the plain - // scan, so the plain scan's map onto them one for one. Which of the two the labelled - // endpoint picks is the hop the query wrote: an out-hop arrives at its target, an - // in-hop leaves its source. - Operation* byLabelScan = nullptr; + return rewriter.create(loc, types, scanLabels._source, scanLabels._target); +} + +void fuseScanEdgesByEndpointLabel(EndpointLabelledEdgeScan& labelledScan, + const DBPassContext& context, + mlir::RewriterBase& rewriter) { + Operation* const scan = labelledScan._scan; + const ArrayAttr endLabels = rewriter.getArrayAttr(labelledScan._endLabels); + + EdgeScanLabels& scanLabels = labelledScan._scanLabels; if (labelledScan._labelledTarget) { - byLabelScan = builder.create(scan.getLoc(), - scan.getSrcids().getType(), - scan.getEids().getType(), - scan.getEtypes().getType(), - scan.getTgtids().getType(), - labelledScan._labels); + scanLabels._target = endLabels; } else { - byLabelScan = builder.create(scan.getLoc(), - scan.getSrcids().getType(), - scan.getEids().getType(), - scan.getEtypes().getType(), - scan.getTgtids().getType(), - labelledScan._labels); + scanLabels._source = endLabels; } - scanOp->replaceAllUsesWith(byLabelScan); + const bool labelsBothEnds = scanLabels._source && scanLabels._target; + if (labelsBothEnds) { + scanLabels._indexedByTarget = indexesByTarget(scanLabels, context); + } - const Operation::operand_range columns = filter.getColumnsToFilter(); - const mlir::ResultRange filtered = filter.getFilteredColumns(); - for (size_t index = 0; index < filtered.size(); index++) { - filtered[index].replaceAllUsesWith(columns[index]); + if (labelledScan._endWasLabelled && !labelsBothEnds) { + rewriter.modifyOpInPlace(scan, [scan, endLabels]() { + scan->setAttr("labels", endLabels); + }); + } else { + rewriter.setInsertionPoint(scan); + Operation* const labelledEdgeScan = createLabelledEdgeScan(scan, scanLabels, rewriter); + rewriter.replaceOp(scan, labelledEdgeScan->getResults()); } - filter.erase(); - eraseIfUnused(labelledScan._check); - eraseIfUnused(labelledScan._labelSet); - scanOp->erase(); + FilterOp filter = labelledScan._filter; + rewriter.replaceOp(filter, filter.getColumnsToFilter()); + + eraseIfUnused(labelledScan._check.getOperation(), rewriter); + eraseIfUnused(labelledScan._labelSet.getOperation(), rewriter); } struct FuseScanEdgesByEndpointLabel : public impl::FuseScanEdgesByEndpointLabelBase { + FuseScanEdgesByEndpointLabel() {} + + FuseScanEdgesByEndpointLabel(const DBPassContext* context) + : _context(context) + { + } + void runOnOperation() override { - mlir::OpBuilder builder(&getContext()); - runFilterPass(getOperation(), - matchEndpointLabelledEdgeScan, - fuseScanEdgesByEndpointLabel, - builder); + const DBPassContext& context = *_context; + const auto fuse = [&context](EndpointLabelledEdgeScan& labelledScan, mlir::RewriterBase& rewriter) { + fuseScanEdgesByEndpointLabel(labelledScan, context, rewriter); + }; + + runFilterWorklist(getOperation(), matchEndpointLabelledEdgeScan, fuse); } + +private: + const DBPassContext* _context {&defaultPassContext}; }; // A hop whose rows are then cut down to the edges the endpoint it reaches carries a set of @@ -3966,14 +4105,11 @@ void collectKnownLabels(Value column, llvm::DenseSet& labels) { addLabelNames(*scanLabels, labels); } return; - } else if (isa(def)) { - if (resultIndex == srcResultIndex) { - addLabelNames(def->getAttrOfType("labels"), labels); - } - return; - } else if (isa(def)) { - if (resultIndex == tgtResultIndex) { - addLabelNames(def->getAttrOfType("labels"), labels); + } else if (EdgeScanLabels scanLabels; readEdgeScanLabels(def, scanLabels)) { + if (resultIndex == srcResultIndex && scanLabels._source) { + addLabelNames(scanLabels._source, labels); + } else if (resultIndex == tgtResultIndex && scanLabels._target) { + addLabelNames(scanLabels._target, labels); } return; } else if (FilterOp filter = dyn_cast(def)) { @@ -7055,24 +7191,6 @@ bool factorHoldsAProduct(Region& factor) { return walked.wasInterrupted(); } -// The label set a scan's label names stand for, resolved against the schema. -void collectScanLabels(ArrayAttr labelNames, const ::db::GraphMetadata& metadata, ::db::LabelSet& labels) { - const ::db::LabelMap& labelMap = metadata.labels(); - for (const Attribute labelAttr : labelNames) { - const StringAttr nameAttr = dyn_cast(labelAttr); - if (!nameAttr) { - continue; - } - - const llvm::StringRef name = nameAttr.getValue(); - const std::optional<::db::LabelID> label = labelMap.get(std::string_view(name.data(), name.size())); - - if (label) { - labels.set(*label); - } - } -} - // The label set the rows of one side are scanned under, when the db level knows one: the // labels of a by-label node scan, and none at all otherwise - every node of the graph, // which is what the v2 planner estimates a variable with no label constraint at. @@ -7168,23 +7286,6 @@ size_t multiplySaturating(size_t rows, size_t factor) { return rows * factor; } -// The labels of whichever of the four by-label edge scans this is, and null for any other -// op. They all read the edges hanging off the nodes the labels select, so they are counted -// the same way whichever endpoint carries them. -ArrayAttr byLabelEdgeScanLabels(Operation* op) { - if (ScanOutEdgesByLabelSrc byLabel = dyn_cast(op)) { - return byLabel.getLabels(); - } else if (ScanInEdgesByLabelTgt byLabel = dyn_cast(op)) { - return byLabel.getLabels(); - } else if (ScanOutEdgesByLabelTgt byLabel = dyn_cast(op)) { - return byLabel.getLabels(); - } else if (ScanInEdgesByLabelSrc byLabel = dyn_cast(op)) { - return byLabel.getLabels(); - } - - return nullptr; -} - // The rows an op seeds a factor with, and nothing at all when it seeds none - a fetch, a // filter or a hop reads a column rather than making one. A listed set of IDs is its own // count, a literal list one row per element, a by-label scan what the graph holds under @@ -7198,31 +7299,36 @@ std::optional estimateSourceRows(Operation* op, } else if (UnwindConst unwind = dyn_cast(op)) { return unwind.getElements().size(); } else if (ScanNodesByLabel byLabel = dyn_cast(op)) { - ::db::LabelSet labels; - collectScanLabels(byLabel.getLabels(), metadata, labels); - - return estimation.estimateNodeCount(labels); + return estimateLabelledNodes(byLabel.getLabels(), metadata, estimation); } else if (isa(op)) { return estimation.estimateNodeCount(::db::LabelSet {}); - } else if (const ArrayAttr scanLabels = byLabelEdgeScanLabels(op)) { + } else if (EdgeScanLabels scanLabels; readEdgeScanLabels(op, scanLabels)) { const size_t nodeCount = estimation.estimateNodeCount(::db::LabelSet {}); if (nodeCount == 0) { return 0; } - ::db::LabelSet labels; - collectScanLabels(scanLabels, metadata, labels); + const bool indexedByTarget = scanLabels._indexedByTarget; + const ArrayAttr indexedLabels = indexedByTarget ? scanLabels._target : scanLabels._source; + const ArrayAttr otherLabels = indexedByTarget ? scanLabels._source : scanLabels._target; - // The edges hanging off those nodes, read as the by-label scan and hop it fused - // were: the node count the labels select, at the graph's average degree. - return multiplySaturating(estimation.estimateNodeCount(labels), estimation.estimateEdgeCount()) / nodeCount; + // The other end's labels keep the share of the walked edges they select of the nodes, + // the two ends' labels taken as independent. + size_t rows = estimation.estimateEdgeCount(); + if (indexedLabels) { + rows = estimateIndexedEdges(indexedLabels, indexedByTarget, metadata, estimation); + } + + if (otherLabels) { + rows = multiplySaturating(estimateLabelledNodes(otherLabels, metadata, estimation), rows) / nodeCount; + } + + return rows; } else if (ScanEdgesByType byType = dyn_cast(op)) { if (byType.getEdgeTypes().empty()) { return 0; } - return estimation.estimateEdgeCount(); - } else if (isa(op)) { return estimation.estimateEdgeCount(); } @@ -8914,6 +9020,10 @@ struct CountFromMetadata : public impl::CountFromMetadataBase } +std::unique_ptr createFuseScanEdgesByEndpointLabel(const DBPassContext* context) { + return std::make_unique(context); +} + std::unique_ptr createFuseHashJoin(const DBPassContext* context) { return std::make_unique(context); } diff --git a/query/ir/dialect/db/DBPasses.h b/query/ir/dialect/db/DBPasses.h index 2804419515..0a24fea38c 100644 --- a/query/ir/dialect/db/DBPasses.h +++ b/query/ir/dialect/db/DBPasses.h @@ -23,6 +23,7 @@ namespace mlir::db { // clearing _usesHashJoin fuses none - the two overrides the v2 planner carries as well. // With no graph there is nothing to estimate, so every match fuses. count_from_metadata // reads _hasPendingWrites: the query then sees writes the graph's counts leave out. +// fuse_scan_edges_by_endpoint_label reads the graph to walk the index of the end with fewer nodes. struct DBPassContext { const ::db::GraphView* _view {nullptr}; bool _forcesHashJoin {false}; @@ -30,6 +31,7 @@ struct DBPassContext { bool _hasPendingWrites {false}; }; +std::unique_ptr createFuseScanEdgesByEndpointLabel(const DBPassContext* context); std::unique_ptr createFuseHashJoin(const DBPassContext* context); std::unique_ptr createCountFromMetadata(const DBPassContext* context); diff --git a/query/ir/dialect/db/DBPasses.td b/query/ir/dialect/db/DBPasses.td index 4a1c04f555..6ebeeac9a7 100644 --- a/query/ir/dialect/db/DBPasses.td +++ b/query/ir/dialect/db/DBPasses.td @@ -64,7 +64,7 @@ def FuseScanInEdgesByLabel : Pass<"fuse_scan_in_edges_by_label"> { } def FuseScanEdgesByEndpointLabel : Pass<"fuse_scan_edges_by_endpoint_label"> { - let summary = "Fuse a whole edge scan and the label check on one of its endpoints into a by-label edge scan"; + let summary = "Fold a label check on either endpoint of an edge scan into the labels the scan reads that endpoint by"; let dependentDialects = ["mlir::db::DB"]; } diff --git a/query/ir/dialect/nl/NLOps.cpp b/query/ir/dialect/nl/NLOps.cpp index 9f0577f6d7..4ea1dbcbcd 100644 --- a/query/ir/dialect/nl/NLOps.cpp +++ b/query/ir/dialect/nl/NLOps.cpp @@ -307,6 +307,22 @@ LogicalResult ScanInEdgesByLabelSrc::inferReturnTypes(MLIRContext* context, return success(); } +LogicalResult ScanOutEdgesByLabelSrcTgt::inferReturnTypes(MLIRContext* context, + std::optional location, + ScanOutEdgesByLabelSrcTgt::Adaptor adaptor, + SmallVectorImpl& inferredReturnTypes) { + inferredReturnTypes.push_back(getEdgeIteratorType(context, {})); + return success(); +} + +LogicalResult ScanInEdgesByLabelSrcTgt::inferReturnTypes(MLIRContext* context, + std::optional location, + ScanInEdgesByLabelSrcTgt::Adaptor adaptor, + SmallVectorImpl& inferredReturnTypes) { + inferredReturnTypes.push_back(getEdgeIteratorType(context, {})); + return success(); +} + // An out-edges fetch produces one row of edge chunks per step, then one // filtered chunk per carried column, mirroring db.get_out_edges LogicalResult GetOutEdges::inferReturnTypes(MLIRContext* context, diff --git a/query/ir/dialect/nl/NLOps.td b/query/ir/dialect/nl/NLOps.td index 7dbb89183b..a493507942 100644 --- a/query/ir/dialect/nl/NLOps.td +++ b/query/ir/dialect/nl/NLOps.td @@ -748,6 +748,54 @@ def ScanInEdgesByLabelSrc : NLOp<"scan_in_edges_by_label_src", [Pure, InferTypeO let assemblyFormat = "`(` $labels `)` attr-dict"; } +/** +* ScanOutEdgesByLabelSrcTgt +*/ +def ScanOutEdgesByLabelSrcTgt : NLOp<"scan_out_edges_by_label_src_tgt", [Pure, InferTypeOpAdaptor]> { + let summary = "Create a database iterator over the edges whose two ends carry their labels, read off the source's index"; + let description = [{ + Imperative counterpart of db.scan_out_edges_by_label_src_tgt. Each step of an + enclosing nl.for fills the same four chunks nl.scan_edges fills, but only for the + edges whose source's label set is a superset of `src_labels` and whose target's is + a superset of `tgt_labels`: + + %edges = nl.scan_out_edges_by_label_src_tgt(["Person"], ["Interest"]) + + It walks the index keyed by the source's label set, as + nl.scan_out_edges_by_label_src does, and keeps each edge whose target carries + `tgt_labels`. Both lists are resolved the way an nl.scan_nodes_by_label name is, + and a label absent from the schema in either leaves the scan unsatisfiable, so the + loop yields no chunk. The result is the same four-chunk edge iterator + nl.scan_edges produces and is inferred, not spelled out. + }]; + let arguments = (ins LabelRefArrayAttr:$src_labels, LabelRefArrayAttr:$tgt_labels); + let results = (outs NLIterator:$result); + let assemblyFormat = "`(` $src_labels `,` $tgt_labels `)` attr-dict"; +} + +/** +* ScanInEdgesByLabelSrcTgt +*/ +def ScanInEdgesByLabelSrcTgt : NLOp<"scan_in_edges_by_label_src_tgt", [Pure, InferTypeOpAdaptor]> { + let summary = "Create a database iterator over the edges whose two ends carry their labels, read off the target's index"; + let description = [{ + Imperative counterpart of db.scan_in_edges_by_label_src_tgt: the rows of + nl.scan_out_edges_by_label_src_tgt, read off the index keyed by the target's label + set, as nl.scan_in_edges_by_label_tgt reads them, keeping each edge whose source + carries `src_labels`: + + %edges = nl.scan_in_edges_by_label_src_tgt(["Interest"], ["Person"]) + + The chunks carry the edge as the graph stores it, so `srcids` holds the source and + `tgtids` the target, as under nl.get_in_edges. Both lists are resolved the way an + nl.scan_nodes_by_label name is, and a label absent from the schema in either leaves + the scan unsatisfiable, so the loop yields no chunk. + }]; + let arguments = (ins LabelRefArrayAttr:$src_labels, LabelRefArrayAttr:$tgt_labels); + let results = (outs NLIterator:$result); + let assemblyFormat = "`(` $src_labels `,` $tgt_labels `)` attr-dict"; +} + /** * GetOutEdges */ diff --git a/query/ir/exec/NLExecutor.cpp b/query/ir/exec/NLExecutor.cpp index fdadc88566..55aedf4e3a 100644 --- a/query/ir/exec/NLExecutor.cpp +++ b/query/ir/exec/NLExecutor.cpp @@ -39,6 +39,7 @@ #include "iterators/PathHopFilter.h" #include "iterators/PathLabelHopFilter.h" #include "iterators/ReversedPathExplorator.h" +#include "iterators/ScanEdgesByEndpointLabelsChunkWriter.h" #include "iterators/ScanEdgesByTypeIterator.h" #include "iterators/ScanEdgesIterator.h" #include "iterators/ScanInEdgesByTargetLabelIterator.h" @@ -6798,6 +6799,68 @@ void NLExecutor::runScanEdgesByTargetLabelLoop(NLExecutionContext* context, NLFu runScanLoopSteps(context, &chunkWriter, &pendingEdges, sources, loopBody, limit); } +void NLExecutor::runScanEdgesByEndpointLabelsFromSourceLoop(NLExecutionContext* context, NLFunctionData* data) { + NLScanEdgesByEndpointLabelsLoopData* loopData = static_cast(data); + + if (!loopData->isMatchable()) { + return; + } + + const NLStmtContainer* loopBody = loopData->getStmts(); + ColumnNodeIDs* sources = loopData->getSources(); + + const NLLimitState* limit = loopData->getLimit(); + + const LabelSet& sourceLabelSet = loopData->getLabelSet(); + const LabelSet& targetLabelSet = loopData->getOtherEndLabelSet(); + const LabelSetHandle sourceLabels(sourceLabelSet); + const LabelSetHandle targetLabels(targetLabelSet); + + ScanOutEdgesByBothLabelsChunkWriter chunkWriter(*context->getView(), sourceLabels, targetLabels); + chunkWriter.setSrcIDs(sources); + chunkWriter.setEdgeIDs(loopData->getEdgeIDs()); + chunkWriter.setEdgeTypes(loopData->getEdgeTypes()); + chunkWriter.setTgtIDs(loopData->getTargets()); + + NLPendingEdgeScan pendingEdges(context, loopData); + pendingEdges.setSourceLabelSet(sourceLabelSet); + pendingEdges.setTargetLabelSet(targetLabelSet); + + runScanLoopSteps(context, &chunkWriter, &pendingEdges, sources, loopBody, limit); +} + +void NLExecutor::runScanEdgesByEndpointLabelsFromTargetLoop(NLExecutionContext* context, NLFunctionData* data) { + NLScanEdgesByEndpointLabelsLoopData* loopData = static_cast(data); + + if (!loopData->isMatchable()) { + return; + } + + const NLStmtContainer* loopBody = loopData->getStmts(); + ColumnNodeIDs* sources = loopData->getSources(); + + const NLLimitState* limit = loopData->getLimit(); + + const LabelSet& targetLabelSet = loopData->getLabelSet(); + const LabelSet& sourceLabelSet = loopData->getOtherEndLabelSet(); + const LabelSetHandle targetLabels(targetLabelSet); + const LabelSetHandle sourceLabels(sourceLabelSet); + + // The writer fills the indexed end through setSrcIDs, which here is the edge's target, + // so the two node columns cross as in runScanEdgesByTargetLabelLoop. + ScanInEdgesByBothLabelsChunkWriter chunkWriter(*context->getView(), targetLabels, sourceLabels); + chunkWriter.setSrcIDs(loopData->getTargets()); + chunkWriter.setEdgeIDs(loopData->getEdgeIDs()); + chunkWriter.setEdgeTypes(loopData->getEdgeTypes()); + chunkWriter.setTgtIDs(sources); + + NLPendingEdgeScan pendingEdges(context, loopData); + pendingEdges.setTargetLabelSet(targetLabelSet); + pendingEdges.setSourceLabelSet(sourceLabelSet); + + runScanLoopSteps(context, &chunkWriter, &pendingEdges, sources, loopBody, limit); +} + void NLExecutor::runGetOutEdgesLoop(NLExecutionContext* context, NLFunctionData* data) { NLEdgeLoopData* loopData = static_cast(data); const ColumnNodeIDs* inputNodeIDs = loopData->getInput(); diff --git a/query/ir/exec/NLExecutor.h b/query/ir/exec/NLExecutor.h index 8cec5d841b..d52226baf5 100644 --- a/query/ir/exec/NLExecutor.h +++ b/query/ir/exec/NLExecutor.h @@ -85,6 +85,8 @@ class NLExecutor { // target-labelled one the index keyed by the target's. static void runScanEdgesBySourceLabelLoop(NLExecutionContext* context, NLFunctionData* data); static void runScanEdgesByTargetLabelLoop(NLExecutionContext* context, NLFunctionData* data); + static void runScanEdgesByEndpointLabelsFromSourceLoop(NLExecutionContext* context, NLFunctionData* data); + static void runScanEdgesByEndpointLabelsFromTargetLoop(NLExecutionContext* context, NLFunctionData* data); static void runGetOutEdgesLoop(NLExecutionContext* context, NLFunctionData* data); static void runGetInEdgesLoop(NLExecutionContext* context, NLFunctionData* data); diff --git a/query/ir/exec/NLPendingEdges.cpp b/query/ir/exec/NLPendingEdges.cpp index c9dbac827f..33969f3f1d 100644 --- a/query/ir/exec/NLPendingEdges.cpp +++ b/query/ir/exec/NLPendingEdges.cpp @@ -244,13 +244,11 @@ NLPendingEdgeScan::~NLPendingEdgeScan() { } void NLPendingEdgeScan::setSourceLabelSet(const LabelSet& labelset) { - _endpointLabels = LabelSetHandle(labelset); - _labelsTheTarget = false; + _sourceLabels = LabelSetHandle(labelset); } void NLPendingEdgeScan::setTargetLabelSet(const LabelSet& labelset) { - _endpointLabels = LabelSetHandle(labelset); - _labelsTheTarget = true; + _targetLabels = LabelSetHandle(labelset); } void NLPendingEdgeScan::fill(size_t maxCount) { @@ -291,13 +289,13 @@ void NLPendingEdgeScan::fill(size_t maxCount) { bool NLPendingEdgeScan::keeps(const CommitWriteBuffer::PendingEdge& edge) const { if (_edgeTypes && !edgeTypeMatches(*_edgeTypes, edge.edgeType)) { return false; + } else if (_sourceLabels.isValid() && !carriesLabels(_writeBuffer, *_view, edge.src, _sourceLabels)) { + return false; + } else if (_targetLabels.isValid() && !carriesLabels(_writeBuffer, *_view, edge.tgt, _targetLabels)) { + return false; } - if (!_endpointLabels.isValid()) { - return true; - } - - return carriesLabels(_writeBuffer, *_view, _labelsTheTarget ? edge.tgt : edge.src, _endpointLabels); + return true; } void NLPendingEdgeScan::clearChunks() { diff --git a/query/ir/exec/NLPendingEdges.h b/query/ir/exec/NLPendingEdges.h index e052fe8234..9b8b6a0f6c 100644 --- a/query/ir/exec/NLPendingEdges.h +++ b/query/ir/exec/NLPendingEdges.h @@ -139,9 +139,8 @@ class NLPendingEdgeScan { void setEdgeTypes(std::span edgeTypes) { _edgeTypes = edgeTypes; } - // The label set one endpoint of the edge must carry at least for the scan to keep it - - // its source for an out-edge scan, its target for an in-edge scan. The label set is - // borrowed, so it must outlive the scan. + // The label set the edge's source, or its target, must carry at least for the scan to + // keep it. The label set is borrowed, so it must outlive the scan. void setSourceLabelSet(const LabelSet& labelset); void setTargetLabelSet(const LabelSet& labelset); @@ -165,8 +164,8 @@ class NLPendingEdgeScan { size_t _edge {0}; std::optional> _edgeTypes; - LabelSetHandle _endpointLabels; - bool _labelsTheTarget {false}; + LabelSetHandle _sourceLabels; + LabelSetHandle _targetLabels; void clearChunks(); diff --git a/query/ir/exec/NLProgram.h b/query/ir/exec/NLProgram.h index 2196d69935..44c916f84e 100644 --- a/query/ir/exec/NLProgram.h +++ b/query/ir/exec/NLProgram.h @@ -774,6 +774,28 @@ class NLScanEdgesByLabelLoopData : public NLScanEdgesLoopData { bool _matchable {true}; }; +// Loop data of the two both-ends edge scans: the by-label scan's, keyed by the label set +// of the end whose index it reads, with the label set the other end must carry. +class NLScanEdgesByEndpointLabelsLoopData : public NLScanEdgesByLabelLoopData { +public: + NLScanEdgesByEndpointLabelsLoopData(ColumnNodeIDs* sources, + ColumnEdgeIDs* edgeIDs, + ColumnEdgeTypes* edgeTypes, + ColumnNodeIDs* targets, + const LabelSet& indexedLabelSet, + const LabelSet& otherEndLabelSet, + bool matchable) + : NLScanEdgesByLabelLoopData(sources, edgeIDs, edgeTypes, targets, indexedLabelSet, matchable), + _otherEndLabelSet(otherEndLabelSet) + { + } + + const LabelSet& getOtherEndLabelSet() const { return _otherEndLabelSet; } + +private: + LabelSet _otherEndLabelSet; +}; + // The seed-expansion core shared by every loop that fans an input node chunk out: the // input, the seed and end node chunks a step fills, the row-to-input-row indices that // gather the carry set, the limit and the body diff --git a/query/ir/exec/NLTranslator.cpp b/query/ir/exec/NLTranslator.cpp index ce80ed5ac8..29f01edacd 100644 --- a/query/ir/exec/NLTranslator.cpp +++ b/query/ir/exec/NLTranslator.cpp @@ -798,6 +798,16 @@ void NLTranslator::translateBlock(mlir::Block& block, NLStmtContainer* body) { bindScanEdgesByLabel(scanInEdgesByLabelTgt, IteratorKind::ScanEdgesByTargetLabel); } else if (nl::ScanOutEdgesByLabelTgt scanOutEdgesByLabelTgt = mlir::dyn_cast(operation)) { bindScanEdgesByLabel(scanOutEdgesByLabelTgt, IteratorKind::ScanEdgesByTargetLabel); + } else if (nl::ScanOutEdgesByLabelSrcTgt scanOutEdgesByLabelSrcTgt = mlir::dyn_cast(operation)) { + IteratorConfig config {IteratorKind::ScanEdgesByEndpointLabelsFromSource, {}, {}}; + resolveLabelNames(scanOutEdgesByLabelSrcTgt.getSrcLabels(), config._labels); + resolveLabelNames(scanOutEdgesByLabelSrcTgt.getTgtLabels(), config._otherEndLabels); + _iteratorConfigs[scanOutEdgesByLabelSrcTgt.getResult()] = config; + } else if (nl::ScanInEdgesByLabelSrcTgt scanInEdgesByLabelSrcTgt = mlir::dyn_cast(operation)) { + IteratorConfig config {IteratorKind::ScanEdgesByEndpointLabelsFromTarget, {}, {}}; + resolveLabelNames(scanInEdgesByLabelSrcTgt.getTgtLabels(), config._labels); + resolveLabelNames(scanInEdgesByLabelSrcTgt.getSrcLabels(), config._otherEndLabels); + _iteratorConfigs[scanInEdgesByLabelSrcTgt.getResult()] = config; } else if (nl::GetOutEdges getOutEdges = mlir::dyn_cast(operation)) { IteratorConfig config {IteratorKind::GetOutEdges, getOutEdges.getInputNodes(), {}}; const mlir::OperandRange carriedColumns = getOutEdges.getColumnsToFilter(); @@ -1237,6 +1247,10 @@ void NLTranslator::translateFor(nl::For forLoop, NLStmtContainer* body) { translateScanEdgesByLabelLoop(config, loopBody, limit, body, &NLExecutor::runScanEdgesBySourceLabelLoop); } else if (config._kind == IteratorKind::ScanEdgesByTargetLabel) { translateScanEdgesByLabelLoop(config, loopBody, limit, body, &NLExecutor::runScanEdgesByTargetLabelLoop); + } else if (config._kind == IteratorKind::ScanEdgesByEndpointLabelsFromSource) { + translateScanEdgesByEndpointLabelsLoop(config, loopBody, limit, body, &NLExecutor::runScanEdgesByEndpointLabelsFromSourceLoop); + } else if (config._kind == IteratorKind::ScanEdgesByEndpointLabelsFromTarget) { + translateScanEdgesByEndpointLabelsLoop(config, loopBody, limit, body, &NLExecutor::runScanEdgesByEndpointLabelsFromTargetLoop); } else if (config._kind == IteratorKind::Sort) { translateSortLoop(config, loopBody, limit, body); } else if (config._kind == IteratorKind::GroupAggregate) { @@ -2016,6 +2030,37 @@ void NLTranslator::translateScanEdgesByLabelLoop(const IteratorConfig& config, translateBlock(loopBody, loopData->getStmts()); } +void NLTranslator::translateScanEdgesByEndpointLabelsLoop(const IteratorConfig& config, + mlir::Block& loopBody, + NLLimitState* limit, + NLStmtContainer* body, + NLHandlerFunction executor) { + ColumnNodeIDs* sources = static_cast(allocColumn(loopBody.getArgument(0))); + ColumnEdgeIDs* edgeIDs = static_cast(allocColumn(loopBody.getArgument(1))); + ColumnEdgeTypes* edgeTypes = static_cast(allocColumn(loopBody.getArgument(2))); + ColumnNodeIDs* targets = static_cast(allocColumn(loopBody.getArgument(3))); + + LabelSet indexedLabelSet; + const bool indexedMatchable = resolveLabelSet(config._labels, indexedLabelSet); + + LabelSet otherEndLabelSet; + const bool otherEndMatchable = resolveLabelSet(config._otherEndLabels, otherEndLabelSet); + + NLScanEdgesByEndpointLabelsLoopData* loopData = + _program->allocFunctionData(sources, + edgeIDs, + edgeTypes, + targets, + indexedLabelSet, + otherEndLabelSet, + indexedMatchable && otherEndMatchable); + loopData->setLimit(limit); + + body->addStmt(NLFunctionDescriptor {executor, loopData}); + + translateBlock(loopBody, loopData->getStmts()); +} + std::optional NLTranslator::findPropertyType(llvm::StringRef name) const { const std::string_view propertyName(name.data(), name.size()); diff --git a/query/ir/exec/NLTranslator.h b/query/ir/exec/NLTranslator.h index c99c9f6ded..8fa8255749 100644 --- a/query/ir/exec/NLTranslator.h +++ b/query/ir/exec/NLTranslator.h @@ -67,6 +67,8 @@ class NLTranslator { ScanEdgesByType, ScanEdgesBySourceLabel, ScanEdgesByTargetLabel, + ScanEdgesByEndpointLabelsFromSource, + ScanEdgesByEndpointLabelsFromTarget, GetOutEdges, GetInEdges, GetEdges, @@ -147,6 +149,10 @@ class NLTranslator { // parameter map's string instead, which the caller keeps alive as long. llvm::SmallVector _labels; + // The label names the end a both-ends edge scan does not read the index of must + // carry, where _labels holds the indexed end's; empty for the other kinds. + llvm::SmallVector _otherEndLabels; + // The edge type names a ScanEdgesByType / GetOutEdgesByType / GetInEdgesByType // iterator filters by, as a disjunction; empty for the other kinds. Like // _labels, views into the op's interned StringAttr storage or into the @@ -465,6 +471,7 @@ class NLTranslator { void translateScanEdgesLoop(mlir::Block& loopBody, NLLimitState* limit, NLStmtContainer* body); void translateScanEdgesByTypeLoop(const IteratorConfig& config, mlir::Block& loopBody, NLLimitState* limit, NLStmtContainer* body); void translateScanEdgesByLabelLoop(const IteratorConfig& config, mlir::Block& loopBody, NLLimitState* limit, NLStmtContainer* body, NLHandlerFunction executor); + void translateScanEdgesByEndpointLabelsLoop(const IteratorConfig& config, mlir::Block& loopBody, NLLimitState* limit, NLStmtContainer* body, NLHandlerFunction executor); // Record the iterator config of one of the four by-label edge scans: they differ in // which endpoint carries the labels, which the kind names, and each reads its label diff --git a/query/ir/lowering/DBLowering.cpp b/query/ir/lowering/DBLowering.cpp index 12d012b457..788057493c 100644 --- a/query/ir/lowering/DBLowering.cpp +++ b/query/ir/lowering/DBLowering.cpp @@ -954,6 +954,8 @@ bool opensSourceLoop(mlir::Operation* operation) { mlir::db::ScanInEdgesByLabelTgt, mlir::db::ScanOutEdgesByLabelTgt, mlir::db::ScanInEdgesByLabelSrc, + mlir::db::ScanOutEdgesByLabelSrcTgt, + mlir::db::ScanInEdgesByLabelSrcTgt, mlir::db::GetOutEdges, mlir::db::GetInEdges, mlir::db::GetEdges, @@ -1390,6 +1392,10 @@ void DBLowering::lowerOperationByKind(mlir::Operation& operation) { lowerScanOutEdgesByLabelTgt(scanOutEdgesByLabelTgt); } else if (mlir::db::ScanInEdgesByLabelSrc scanInEdgesByLabelSrc = mlir::dyn_cast(operation)) { lowerScanInEdgesByLabelSrc(scanInEdgesByLabelSrc); + } else if (mlir::db::ScanOutEdgesByLabelSrcTgt scanOutEdgesByLabelSrcTgt = mlir::dyn_cast(operation)) { + lowerScanOutEdgesByLabelSrcTgt(scanOutEdgesByLabelSrcTgt); + } else if (mlir::db::ScanInEdgesByLabelSrcTgt scanInEdgesByLabelSrcTgt = mlir::dyn_cast(operation)) { + lowerScanInEdgesByLabelSrcTgt(scanInEdgesByLabelSrcTgt); } else if (mlir::db::GetOutEdges getOutEdges = mlir::dyn_cast(operation)) { lowerGetOutEdges(getOutEdges); } else if (mlir::db::GetInEdges getInEdges = mlir::dyn_cast(operation)) { @@ -2681,6 +2687,24 @@ void DBLowering::lowerScanInEdgesByLabelSrc(mlir::db::ScanInEdgesByLabelSrc scan buildLoopForSource(edges.getResult(), scanInEdgesByLabelSrc.getOperation()); } +void DBLowering::lowerScanOutEdgesByLabelSrcTgt(mlir::db::ScanOutEdgesByLabelSrcTgt scanOutEdgesByLabelSrcTgt) { + setInsertionInto(_rootBlock); + + nl::ScanOutEdgesByLabelSrcTgt edges = _builder.create(_builder.getUnknownLoc(), + scanOutEdgesByLabelSrcTgt.getSrcLabelsAttr(), + scanOutEdgesByLabelSrcTgt.getTgtLabelsAttr()); + buildLoopForSource(edges.getResult(), scanOutEdgesByLabelSrcTgt.getOperation()); +} + +void DBLowering::lowerScanInEdgesByLabelSrcTgt(mlir::db::ScanInEdgesByLabelSrcTgt scanInEdgesByLabelSrcTgt) { + setInsertionInto(_rootBlock); + + nl::ScanInEdgesByLabelSrcTgt edges = _builder.create(_builder.getUnknownLoc(), + scanInEdgesByLabelSrcTgt.getSrcLabelsAttr(), + scanInEdgesByLabelSrcTgt.getTgtLabelsAttr()); + buildLoopForSource(edges.getResult(), scanInEdgesByLabelSrcTgt.getOperation()); +} + void DBLowering::lowerGetOutEdges(mlir::db::GetOutEdges getOutEdges) { // Map the input node column and the carry set to the nl chunks they lowered // to. The fetch nests in the loop that binds its input chunk. diff --git a/query/ir/lowering/DBLowering.h b/query/ir/lowering/DBLowering.h index 8655d9cbf3..2a2da1166d 100644 --- a/query/ir/lowering/DBLowering.h +++ b/query/ir/lowering/DBLowering.h @@ -325,6 +325,8 @@ class DBLowering { void lowerScanInEdgesByLabelTgt(mlir::db::ScanInEdgesByLabelTgt scanInEdgesByLabelTgt); void lowerScanOutEdgesByLabelTgt(mlir::db::ScanOutEdgesByLabelTgt scanOutEdgesByLabelTgt); void lowerScanInEdgesByLabelSrc(mlir::db::ScanInEdgesByLabelSrc scanInEdgesByLabelSrc); + void lowerScanOutEdgesByLabelSrcTgt(mlir::db::ScanOutEdgesByLabelSrcTgt scanOutEdgesByLabelSrcTgt); + void lowerScanInEdgesByLabelSrcTgt(mlir::db::ScanInEdgesByLabelSrcTgt scanInEdgesByLabelSrcTgt); void lowerGetOutEdges(mlir::db::GetOutEdges getOutEdges); void lowerGetInEdges(mlir::db::GetInEdges getInEdges); void lowerGetEdges(mlir::db::GetEdges getEdges); diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index bab79c5a70..4166dcc823 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -77,6 +77,7 @@ set(storage_sources iterators/ScanEdgeTypesIterator.cpp iterators/ScanPropertyTypesIterator.cpp iterators/ScanOutEdgesBySourceLabelIterator.cpp + iterators/ScanEdgesByEndpointLabelsChunkWriter.cpp iterators/MatchLabelSetIterator.cpp iterators/NeighbourhoodSampleIterator.cpp iterators/PathCycleTable.cpp diff --git a/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.cpp b/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.cpp new file mode 100644 index 0000000000..8758408498 --- /dev/null +++ b/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.cpp @@ -0,0 +1,158 @@ +#include "ScanEdgesByEndpointLabelsChunkWriter.h" + +#include + +#include "reader/GraphReader.h" +#include "IteratorUtils.h" + +#include "BioAssert.h" + +using namespace db; + +namespace { + +constexpr size_t N_COLUMNS = 4; +constexpr size_t N_COMBINATIONS = 1 << N_COLUMNS; + +} + +template +ScanEdgesByEndpointLabelsChunkWriter::ScanEdgesByEndpointLabelsChunkWriter(const GraphView& view, + const LabelSetHandle& indexedLabels, + const LabelSetHandle& otherLabels) + : IndexIterator(view, indexedLabels), + _otherLabels(otherLabels), + _filter(view) +{ +} + +template +ScanEdgesByEndpointLabelsChunkWriter::~ScanEdgesByEndpointLabelsChunkWriter() { +} + +template +void ScanEdgesByEndpointLabelsChunkWriter::filterTombstones() { + // Base column of this ChunkWriter is _edgeIDs + _filter.populateRanges(_edgeIDs); + + _filter.filter(_edgeIDs); + + if (_srcs) { + _filter.filter(_srcs); + } + if (_tgts) { + _filter.filter(_tgts); + } + if (_types) { + _filter.filter(_types); + } + + _filter.reset(); +} + +template +void ScanEdgesByEndpointLabelsChunkWriter::fill(size_t maxCount) { + size_t remainingToMax = maxCount; + static constexpr auto bools = generateArray(); + static constexpr auto masks = generateBitmasks(); + + bioassert(_srcs || _tgts || _edgeIDs || _types, + "ScanEdgesByEndpointLabelsChunkWriter must be initialized with at least one valid column"); + + if (_srcs) { + _srcs->clear(); + } + if (_edgeIDs) { + _edgeIDs->clear(); + } + if (_tgts) { + _tgts->clear(); + } + if (_types) { + _types->clear(); + } + + const GraphReader reader(this->_view); + size_t written = 0; + + const auto fill = [&] conditions>() { + while (this->isValid() && remainingToMax > 0) { + _keptEdges.clear(); + while (this->_edgeIt != this->_edges.end() && _keptEdges.size() < remainingToMax) { + const EdgeRecord& edge = *this->_edgeIt; + const LabelSetHandle otherEndLabels = reader.getNodeLabelSet(edge._otherID); + + if (otherEndLabels.isValid() && otherEndLabels.hasAtLeastLabels(_otherLabels)) { + _keptEdges.push_back(&edge); + } + + ++this->_edgeIt; + } + + const size_t keptCount = _keptEdges.size(); + const size_t newSize = written + keptCount; + + if constexpr (conditions[0]) { + _srcs->resize(newSize); + std::transform(_keptEdges.begin(), + _keptEdges.end(), + _srcs->begin() + written, + [](const EdgeRecord* edge) { return edge->_nodeID; }); + } + if constexpr (conditions[1]) { + _edgeIDs->resize(newSize); + std::transform(_keptEdges.begin(), + _keptEdges.end(), + _edgeIDs->begin() + written, + [](const EdgeRecord* edge) { return edge->_edgeID; }); + } + if constexpr (conditions[2]) { + _tgts->resize(newSize); + std::transform(_keptEdges.begin(), + _keptEdges.end(), + _tgts->begin() + written, + [](const EdgeRecord* edge) { return edge->_otherID; }); + } + if constexpr (conditions[3]) { + _types->resize(newSize); + std::transform(_keptEdges.begin(), + _keptEdges.end(), + _types->begin() + written, + [](const EdgeRecord* edge) { return edge->_edgeTypeID; }); + } + + written = newSize; + remainingToMax -= keptCount; + + if (this->_edgeIt == this->_edges.end()) { + this->nextValid(); + } + } + }; + + switch (bitmask::create(_srcs, _edgeIDs, _tgts, _types)) { + CASE(0); + CASE(1); + CASE(2); + CASE(3); + CASE(4); + CASE(5); + CASE(6); + CASE(7); + CASE(8); + CASE(9); + CASE(10); + CASE(11); + CASE(12); + CASE(13); + CASE(14); + CASE(15); + } + + if (this->_view.hasDeletedEdges()) { + filterTombstones(); + } +} + +template class db::ScanEdgesByEndpointLabelsChunkWriter; +template class db::ScanEdgesByEndpointLabelsChunkWriter; diff --git a/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.h b/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.h new file mode 100644 index 0000000000..429d191748 --- /dev/null +++ b/storage/iterators/ScanEdgesByEndpointLabelsChunkWriter.h @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include "ScanOutEdgesBySourceLabelIterator.h" +#include "ScanInEdgesByTargetLabelIterator.h" +#include "ChunkWriter.h" +#include "TombstoneFilter.h" +#include "columns/ColumnEdgeTypes.h" +#include "columns/ColumnIDs.h" +#include "metadata/LabelSetHandle.h" + +namespace db { + +// Walks the edges a by-label edge index holds under one end's label set and keeps those +// whose other end carries at least _otherLabels. IndexIterator picks the index, so the +// node column it fills through setSrcIDs is the indexed end and setTgtIDs the other end. +template +class ScanEdgesByEndpointLabelsChunkWriter : public IndexIterator { +public: + ScanEdgesByEndpointLabelsChunkWriter(const GraphView& view, + const LabelSetHandle& indexedLabels, + const LabelSetHandle& otherLabels); + ~ScanEdgesByEndpointLabelsChunkWriter() override; + + void fill(size_t maxCount); + + void setSrcIDs(ColumnNodeIDs* srcIDs) { _srcs = srcIDs; } + void setEdgeIDs(ColumnEdgeIDs* edgeIDs) { _edgeIDs = edgeIDs; } + void setTgtIDs(ColumnNodeIDs* tgtIDs) { _tgts = tgtIDs; } + void setEdgeTypes(ColumnEdgeTypes* types) { _types = types; } + +private: + LabelSetHandle _otherLabels; + + ColumnEdgeIDs* _edgeIDs {nullptr}; + ColumnNodeIDs* _srcs {nullptr}; + ColumnNodeIDs* _tgts {nullptr}; + ColumnEdgeTypes* _types {nullptr}; + + TombstoneFilter _filter; + std::vector _keptEdges; + + void filterTombstones(); +}; + +using ScanOutEdgesByBothLabelsChunkWriter = ScanEdgesByEndpointLabelsChunkWriter; +using ScanInEdgesByBothLabelsChunkWriter = ScanEdgesByEndpointLabelsChunkWriter; + +static_assert(SrcIDsChunkWriter); +static_assert(EdgeIDsChunkWriter); +static_assert(TgtIDsChunkWriter); +static_assert(EdgeTypesChunkWriter); +static_assert(SrcIDsChunkWriter); +static_assert(EdgeIDsChunkWriter); +static_assert(TgtIDsChunkWriter); +static_assert(EdgeTypesChunkWriter); + +} diff --git a/storage/reader/GraphReader.cpp b/storage/reader/GraphReader.cpp index a771e712ed..6e43fd2210 100644 --- a/storage/reader/GraphReader.cpp +++ b/storage/reader/GraphReader.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include @@ -21,30 +23,95 @@ namespace rv = ranges::views; namespace { -// Whether a data part after @param partIndex holds this property type at all. Only those -// parts can hold a newer entry overriding one of this part's. -bool holdsPropertyLater(DataPartSpan parts, size_t partIndex, PropertyTypeID propertyTypeID) { - for (size_t newer = partIndex + 1; newer < parts.size(); newer++) { - if (parts[newer]->nodeProperties().hasPropertyType(propertyTypeID)) { - return true; +template +void visitMatchingNodeRanges(DataPartSpan parts, const LabelSetHandle& labelset, VisitRange visitRange) { + for (const auto& part : parts) { + const LabelSetIndexer& indexer = part->nodes().getLabelSetIndexer(); + for (auto labelsetIt = indexer.matchIterate(labelset); labelsetIt.isValid(); labelsetIt.next()) { + visitRange(labelsetIt.getValue()); } } +} - return false; +size_t countIndexedEdges(const LabelSetIndexer& indexer, const LabelSetHandle& labelset) { + size_t count = 0; + for (auto labelsetIt = indexer.matchIterate(labelset); labelsetIt.isValid(); labelsetIt.next()) { + for (const EdgeIndexer::EdgeSpan& span : labelsetIt.getValue()) { + count += span.size(); + } + } + + return count; } -// Whether a data part after @param partIndex holds this property for the entity too -bool isPropertyOverriddenLater(DataPartSpan parts, - size_t partIndex, - PropertyTypeID propertyTypeID, - EntityID entityID) { - for (size_t newer = partIndex + 1; newer < parts.size(); newer++) { - if (parts[newer]->nodeProperties().hasEntry(propertyTypeID, entityID)) { - return true; +// The matched entries of one part that stand for a live node: those whose node is neither +// deleted nor held by an entry of a newer part. The excluded entries are found from the +// smaller side: each matched entry checked against the exclusions, or each exclusion +// checked against the part's entries and the labels. +size_t countLiveEntries(const GraphReader& reader, + const PropertyManager& properties, + PropertyTypeID propertyTypeID, + const LabelSetHandle& labelset, + const std::unordered_set& overridden) { + const LabelSetPropertyIndexer* indexer = properties.tryGetIndexer(propertyTypeID); + bioassert(indexer, "A part holding a property container has no label set indexer for it"); + + size_t matchedCount = 0; + for (auto labelsetIt = indexer->matchIterate(labelset); labelsetIt.isValid(); labelsetIt.next()) { + for (const PropertyRange& range : labelsetIt.getValue()) { + matchedCount += range._count; } } - return false; + const Tombstones& tombstones = reader.getView().tombstones(); + const size_t exclusionCount = tombstones.numNodes() + overridden.size(); + if (exclusionCount == 0) { + return matchedCount; + } + + if (matchedCount <= exclusionCount) { + const std::span entityIDs = properties.ids(propertyTypeID); + + size_t liveCount = 0; + for (auto labelsetIt = indexer->matchIterate(labelset); labelsetIt.isValid(); labelsetIt.next()) { + for (const PropertyRange& range : labelsetIt.getValue()) { + for (const EntityID entityID : entityIDs.subspan(range._offset, range._count)) { + const bool deleted = tombstones.containsNode(NodeID {entityID.getValue()}); + if (!deleted && !overridden.contains(entityID)) { + liveCount++; + } + } + } + } + + return liveCount; + } + + const PropertyContainer& container = *properties.find(propertyTypeID)->second; + const auto holdsMatchedEntry = [&reader, &container, &labelset](NodeID nodeID) { + if (!container.has(EntityID {nodeID.getValue()})) { + return false; + } + + const LabelSetHandle nodeLabels = reader.getNodeLabelSet(nodeID); + return nodeLabels.isValid() && nodeLabels.hasAtLeastLabels(labelset); + }; + + size_t excludedCount = 0; + for (const NodeID nodeID : tombstones.nodeTombstones()) { + if (holdsMatchedEntry(nodeID)) { + excludedCount++; + } + } + + for (const EntityID entityID : overridden) { + const NodeID nodeID {entityID.getValue()}; + if (!tombstones.containsNode(nodeID) && holdsMatchedEntry(nodeID)) { + excludedCount++; + } + } + + return matchedCount - excludedCount; } } @@ -104,31 +171,56 @@ const EdgeRecord* GraphReader::getEdge(EdgeID edgeID) const { } size_t GraphReader::getNodeCountMatchingLabelset(const LabelSetHandle& labelset) const { - // Iterate each data part exactly once, summing the node counts of that - // part's own labelsets that match `labelset`. Every node is counted once, - // in the part that owns it. + const DataPartSpan parts = _view.dataparts(); + size_t count = 0; + visitMatchingNodeRanges(parts, labelset, [&count](NodeRange range) { + count += range._count; + }); + + const Tombstones& tombstones = _view.tombstones(); + if (!tombstones.hasNodes()) { + return count; + } - auto filteredNodeCount = [](NodeRange range, const Tombstones* tombstones) { - size_t count = range._count; - if (tombstones->hasNodes()) { - for (const auto id : range) { - if (tombstones->containsNode(id)) { - count -= 1; + // The deleted nodes are found from the smaller side: each matched node checked against + // the deletions, or each deletion checked against the labels. + size_t deletedCount = 0; + if (count <= tombstones.numNodes()) { + visitMatchingNodeRanges(parts, labelset, [&deletedCount, &tombstones](NodeRange range) { + for (const NodeID nodeID : range) { + if (tombstones.containsNode(nodeID)) { + deletedCount++; } } + }); + } else { + for (const NodeID nodeID : tombstones.nodeTombstones()) { + const LabelSetHandle nodeLabels = getNodeLabelSet(nodeID); + if (nodeLabels.isValid() && nodeLabels.hasAtLeastLabels(labelset)) { + deletedCount++; + } } + } - return count; - }; + return count - deletedCount; +} - const auto* tombstones = &_view.tombstones(); +size_t GraphReader::getOutEdgeCountMatchingLabelset(const LabelSetHandle& labelset) const { + size_t count = 0; for (const auto& part : _view.dataparts()) { - const NodeContainer& nodes = part->nodes(); - const LabelSetIndexer& indexer = nodes.getLabelSetIndexer(); - for (auto it = indexer.matchIterate(labelset); it.isValid(); it.next()) { - count += filteredNodeCount(it.getValue(), tombstones); - } + const EdgeIndexer& indexer = part->edgeIndexer(); + count += countIndexedEdges(indexer.getOutsByLabelSet(), labelset); + } + + return count; +} + +size_t GraphReader::getInEdgeCountMatchingLabelset(const LabelSetHandle& labelset) const { + size_t count = 0; + for (const auto& part : _view.dataparts()) { + const EdgeIndexer& indexer = part->edgeIndexer(); + count += countIndexedEdges(indexer.getInsByLabelSet(), labelset); } return count; @@ -137,40 +229,35 @@ size_t GraphReader::getNodeCountMatchingLabelset(const LabelSetHandle& labelset) size_t GraphReader::getNodeCountWithProperty(const LabelSetHandle& labelset, PropertyTypeID propertyTypeID) const { const DataPartSpan parts = _view.dataparts(); - const Tombstones& tombstones = _view.tombstones(); - size_t count = 0; + size_t oldestHolder = parts.size(); for (size_t partIndex = 0; partIndex < parts.size(); partIndex++) { + if (parts[partIndex]->nodeProperties().hasPropertyType(propertyTypeID)) { + oldestHolder = partIndex; + break; + } + } + + // The entities a newer part holds an entry for, a value or an explicit null. That entry + // stands for the node, so an older part's entry for it no longer counts. + std::unordered_set overridden; + + size_t count = 0; + for (size_t newerParts = parts.size(); newerParts > oldestHolder; newerParts--) { + const size_t partIndex = newerParts - 1; const PropertyManager& properties = parts[partIndex]->nodeProperties(); if (!properties.hasPropertyType(propertyTypeID)) { continue; } - const LabelSetPropertyIndexer* indexer = properties.tryGetIndexer(propertyTypeID); - bioassert(indexer, "A part holding a property container has no label set indexer for it"); - - // An entry is only read to exclude it: its node is deleted, or a later part holds a - // newer entry standing for the same node. With neither possible the indexed ranges - // answer on their own, one addition per range. - const bool excludesEntries = tombstones.hasNodes() || holdsPropertyLater(parts, partIndex, propertyTypeID); - const std::span entityIDs = properties.ids(propertyTypeID); - - for (auto labelsetIt = indexer->matchIterate(labelset); labelsetIt.isValid(); labelsetIt.next()) { - for (const PropertyRange& range : labelsetIt.getValue()) { - if (!excludesEntries) { - count += range._count; - continue; - } + count += countLiveEntries(*this, properties, propertyTypeID, labelset, overridden); - for (const EntityID entityID : entityIDs.subspan(range._offset, range._count)) { - const bool deleted = tombstones.containsNode(NodeID {entityID.getValue()}); - const bool overridden = isPropertyOverriddenLater(parts, partIndex, propertyTypeID, entityID); - - if (!deleted && !overridden) { - count++; - } - } - } + if (partIndex > oldestHolder) { + const PropertyContainer& container = *properties.find(propertyTypeID)->second; + const PropertyContainer::IDs& valueIDs = container.ids(); + const PropertyContainer::IDs& nullIDs = container.nullIds(); + overridden.insert(valueIDs.begin(), valueIDs.end()); + overridden.insert(nullIDs.begin(), nullIDs.end()); } } diff --git a/storage/reader/GraphReader.h b/storage/reader/GraphReader.h index 19348785b7..602279348b 100644 --- a/storage/reader/GraphReader.h +++ b/storage/reader/GraphReader.h @@ -76,6 +76,14 @@ class GraphReader { [[nodiscard]] LabelSetHandle getNodeLabelSet(NodeID nodeID) const; [[nodiscard]] size_t getNodeCountMatchingLabelset(const LabelSetHandle& labelset) const; + /** + * @brief Returns the number of edges the out-edge (in-edge) index holds under the label + * sets matching @p labelset, which is the number of edges a scan of that index walks. + * @detail Counts the edges a tombstone deleted, since the walk visits them too. + */ + [[nodiscard]] size_t getOutEdgeCountMatchingLabelset(const LabelSetHandle& labelset) const; + [[nodiscard]] size_t getInEdgeCountMatchingLabelset(const LabelSetHandle& labelset) const; + /** * @brief Returns the number of nodes matching @p labelset which hold property @p * propertyTypeID, counting each node once and none that a tombstone deleted. diff --git a/test/query-test-suite/tests/join-comma-shared-variable.json b/test/query-test-suite/tests/join-comma-shared-variable.json index bae502b115..e5fb0feca2 100644 --- a/test/query-test-suite/tests/join-comma-shared-variable.json +++ b/test/query-test-suite/tests/join-comma-shared-variable.json @@ -3,8 +3,8 @@ "graph": "simpledb", "query": "MATCH (a:Person)-->(i1:Interest), (a)-->(i2:Interest) WHERE i1.name <> i2.name RETURN a.name, i1.name, i2.name", "expect": { - "result": "a.name,i1.name,i2.name\nRemy,Computers,Ghosts\nRemy,Eighties,Ghosts\nRemy,Ghosts,Computers\nRemy,Eighties,Computers\nRemy,Ghosts,Eighties\nRemy,Computers,Eighties\nAdam,Cooking,Bio\nAdam,Bio,Cooking\nMaxime,Padel,Bio\nMaxime,Bio,Padel\nLuc,Computers,Animals\nLuc,Animals,Computers\nCyrus,Travel,Gym\nCyrus,Gym,Travel\nSuhas,JiuJitsu,Gym\nSuhas,Gym,JiuJitsu", - "resultJson": "{\"header\":{\"column_names\":[\"a.name\",\"i1.name\",\"i2.name\"],\"column_types\":[\"String\",\"String\",\"String\"]},\"data\":[[[\"Remy\",\"Remy\",\"Remy\",\"Remy\",\"Remy\",\"Remy\",\"Adam\",\"Adam\",\"Maxime\",\"Maxime\",\"Luc\",\"Luc\",\"Cyrus\",\"Cyrus\",\"Suhas\",\"Suhas\"],[\"Computers\",\"Eighties\",\"Ghosts\",\"Eighties\",\"Ghosts\",\"Computers\",\"Cooking\",\"Bio\",\"Padel\",\"Bio\",\"Computers\",\"Animals\",\"Travel\",\"Gym\",\"JiuJitsu\",\"Gym\"],[\"Ghosts\",\"Ghosts\",\"Computers\",\"Computers\",\"Eighties\",\"Eighties\",\"Bio\",\"Cooking\",\"Bio\",\"Padel\",\"Animals\",\"Computers\",\"Gym\",\"Travel\",\"Gym\",\"JiuJitsu\"]]]}" + "result": "a.name,i1.name,i2.name\nRemy,Ghosts,Computers\nRemy,Eighties,Computers\nRemy,Ghosts,Eighties\nRemy,Computers,Eighties\nAdam,Cooking,Bio\nAdam,Bio,Cooking\nRemy,Computers,Ghosts\nRemy,Eighties,Ghosts\nMaxime,Padel,Bio\nMaxime,Bio,Padel\nLuc,Animals,Computers\nLuc,Computers,Animals\nCyrus,Travel,Gym\nCyrus,Gym,Travel\nSuhas,JiuJitsu,Gym\nSuhas,Gym,JiuJitsu", + "resultJson": "{\"header\":{\"column_names\":[\"a.name\",\"i1.name\",\"i2.name\"],\"column_types\":[\"String\",\"String\",\"String\"]},\"data\":[[[\"Remy\",\"Remy\",\"Remy\",\"Remy\",\"Adam\",\"Adam\",\"Remy\",\"Remy\",\"Maxime\",\"Maxime\",\"Luc\",\"Luc\",\"Cyrus\",\"Cyrus\",\"Suhas\",\"Suhas\"],[\"Ghosts\",\"Eighties\",\"Ghosts\",\"Computers\",\"Cooking\",\"Bio\",\"Computers\",\"Eighties\",\"Padel\",\"Bio\",\"Animals\",\"Computers\",\"Travel\",\"Gym\",\"JiuJitsu\",\"Gym\"],[\"Computers\",\"Computers\",\"Eighties\",\"Eighties\",\"Bio\",\"Cooking\",\"Ghosts\",\"Ghosts\",\"Bio\",\"Padel\",\"Computers\",\"Animals\",\"Gym\",\"Travel\",\"Gym\",\"JiuJitsu\"]]]}" }, "tags": [ "join", diff --git a/test/query-test-suite/tests/match-call-wildcard-3.json b/test/query-test-suite/tests/match-call-wildcard-3.json index 156ba9da16..bdf82df837 100644 --- a/test/query-test-suite/tests/match-call-wildcard-3.json +++ b/test/query-test-suite/tests/match-call-wildcard-3.json @@ -3,8 +3,8 @@ "graph": "simpledb", "query": "MATCH (n:Person)-->(m:Interest)\nCALL db.labels() YIELD label, id\nRETURN *, m.name", "expect": { - "result": "n,m,label,id,m.name\n0,6,Person,0,Ghosts\n0,6,SoftwareEngineering,1,Ghosts\n0,6,Founder,2,Ghosts\n0,6,Bioinformatics,3,Ghosts\n0,6,Interest,4,Ghosts\n0,6,Exotic,5,Ghosts\n0,6,Supernatural,6,Ghosts\n0,6,SleepDisturber,7,Ghosts\n0,6,Sales,8,Ghosts\n0,2,Person,0,Computers\n0,2,SoftwareEngineering,1,Computers\n0,2,Founder,2,Computers\n0,2,Bioinformatics,3,Computers\n0,2,Interest,4,Computers\n0,2,Exotic,5,Computers\n0,2,Supernatural,6,Computers\n0,2,SleepDisturber,7,Computers\n0,2,Sales,8,Computers\n0,3,Person,0,Eighties\n0,3,SoftwareEngineering,1,Eighties\n0,3,Founder,2,Eighties\n0,3,Bioinformatics,3,Eighties\n0,3,Interest,4,Eighties\n0,3,Exotic,5,Eighties\n0,3,Supernatural,6,Eighties\n0,3,SleepDisturber,7,Eighties\n0,3,Sales,8,Eighties\n1,4,Person,0,Bio\n1,4,SoftwareEngineering,1,Bio\n1,4,Founder,2,Bio\n1,4,Bioinformatics,3,Bio\n1,4,Interest,4,Bio\n1,4,Exotic,5,Bio\n1,4,Supernatural,6,Bio\n1,4,SleepDisturber,7,Bio\n1,4,Sales,8,Bio\n1,5,Person,0,Cooking\n1,5,SoftwareEngineering,1,Cooking\n1,5,Founder,2,Cooking\n1,5,Bioinformatics,3,Cooking\n1,5,Interest,4,Cooking\n1,5,Exotic,5,Cooking\n1,5,Supernatural,6,Cooking\n1,5,SleepDisturber,7,Cooking\n1,5,Sales,8,Cooking\n8,4,Person,0,Bio\n8,4,SoftwareEngineering,1,Bio\n8,4,Founder,2,Bio\n8,4,Bioinformatics,3,Bio\n8,4,Interest,4,Bio\n8,4,Exotic,5,Bio\n8,4,Supernatural,6,Bio\n8,4,SleepDisturber,7,Bio\n8,4,Sales,8,Bio\n8,7,Person,0,Padel\n8,7,SoftwareEngineering,1,Padel\n8,7,Founder,2,Padel\n8,7,Bioinformatics,3,Padel\n8,7,Interest,4,Padel\n8,7,Exotic,5,Padel\n8,7,Supernatural,6,Padel\n8,7,SleepDisturber,7,Padel\n8,7,Sales,8,Padel\n9,10,Person,0,Animals\n9,10,SoftwareEngineering,1,Animals\n9,10,Founder,2,Animals\n9,10,Bioinformatics,3,Animals\n9,10,Interest,4,Animals\n9,10,Exotic,5,Animals\n9,10,Supernatural,6,Animals\n9,10,SleepDisturber,7,Animals\n9,10,Sales,8,Animals\n9,2,Person,0,Computers\n9,2,SoftwareEngineering,1,Computers\n9,2,Founder,2,Computers\n9,2,Bioinformatics,3,Computers\n9,2,Interest,4,Computers\n9,2,Exotic,5,Computers\n9,2,Supernatural,6,Computers\n9,2,SleepDisturber,7,Computers\n9,2,Sales,8,Computers\n11,5,Person,0,Cooking\n11,5,SoftwareEngineering,1,Cooking\n11,5,Founder,2,Cooking\n11,5,Bioinformatics,3,Cooking\n11,5,Interest,4,Cooking\n11,5,Exotic,5,Cooking\n11,5,Supernatural,6,Cooking\n11,5,SleepDisturber,7,Cooking\n11,5,Sales,8,Cooking\n15,13,Person,0,Gym\n15,13,SoftwareEngineering,1,Gym\n15,13,Founder,2,Gym\n15,13,Bioinformatics,3,Gym\n15,13,Interest,4,Gym\n15,13,Exotic,5,Gym\n15,13,Supernatural,6,Gym\n15,13,SleepDisturber,7,Gym\n15,13,Sales,8,Gym\n15,14,Person,0,Travel\n15,14,SoftwareEngineering,1,Travel\n15,14,Founder,2,Travel\n15,14,Bioinformatics,3,Travel\n15,14,Interest,4,Travel\n15,14,Exotic,5,Travel\n15,14,Supernatural,6,Travel\n15,14,SleepDisturber,7,Travel\n15,14,Sales,8,Travel\n12,13,Person,0,Gym\n12,13,SoftwareEngineering,1,Gym\n12,13,Founder,2,Gym\n12,13,Bioinformatics,3,Gym\n12,13,Interest,4,Gym\n12,13,Exotic,5,Gym\n12,13,Supernatural,6,Gym\n12,13,SleepDisturber,7,Gym\n12,13,Sales,8,Gym\n12,16,Person,0,JiuJitsu\n12,16,SoftwareEngineering,1,JiuJitsu\n12,16,Founder,2,JiuJitsu\n12,16,Bioinformatics,3,JiuJitsu\n12,16,Interest,4,JiuJitsu\n12,16,Exotic,5,JiuJitsu\n12,16,Supernatural,6,JiuJitsu\n12,16,SleepDisturber,7,JiuJitsu\n12,16,Sales,8,JiuJitsu\n17,13,Person,0,Gym\n17,13,SoftwareEngineering,1,Gym\n17,13,Founder,2,Gym\n17,13,Bioinformatics,3,Gym\n17,13,Interest,4,Gym\n17,13,Exotic,5,Gym\n17,13,Supernatural,6,Gym\n17,13,SleepDisturber,7,Gym\n17,13,Sales,8,Gym", - "resultJson": "{\"header\":{\"column_names\":[\"n\",\"m\",\"label\",\"id\",\"m.name\"],\"column_types\":[\"UInt64\",\"UInt64\",\"String\",\"UInt64\",\"String\"]},\"data\":[[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,11,11,11,11,11,11,11,11,11,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,17,17,17,17,17,17,17,17,17],[6,6,6,6,6,6,6,6,6,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,4,7,7,7,7,7,7,7,7,7,10,10,10,10,10,10,10,10,10,2,2,2,2,2,2,2,2,2,5,5,5,5,5,5,5,5,5,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,13,13,13,13,13,13,13,13,13,16,16,16,16,16,16,16,16,16,13,13,13,13,13,13,13,13,13],[\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\"],[0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8],[\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\"]]]}" + "result": "n,m,label,id,m.name\n0,2,Person,0,Computers\n0,2,SoftwareEngineering,1,Computers\n0,2,Founder,2,Computers\n0,2,Bioinformatics,3,Computers\n0,2,Interest,4,Computers\n0,2,Exotic,5,Computers\n0,2,Supernatural,6,Computers\n0,2,SleepDisturber,7,Computers\n0,2,Sales,8,Computers\n0,3,Person,0,Eighties\n0,3,SoftwareEngineering,1,Eighties\n0,3,Founder,2,Eighties\n0,3,Bioinformatics,3,Eighties\n0,3,Interest,4,Eighties\n0,3,Exotic,5,Eighties\n0,3,Supernatural,6,Eighties\n0,3,SleepDisturber,7,Eighties\n0,3,Sales,8,Eighties\n1,4,Person,0,Bio\n1,4,SoftwareEngineering,1,Bio\n1,4,Founder,2,Bio\n1,4,Bioinformatics,3,Bio\n1,4,Interest,4,Bio\n1,4,Exotic,5,Bio\n1,4,Supernatural,6,Bio\n1,4,SleepDisturber,7,Bio\n1,4,Sales,8,Bio\n1,5,Person,0,Cooking\n1,5,SoftwareEngineering,1,Cooking\n1,5,Founder,2,Cooking\n1,5,Bioinformatics,3,Cooking\n1,5,Interest,4,Cooking\n1,5,Exotic,5,Cooking\n1,5,Supernatural,6,Cooking\n1,5,SleepDisturber,7,Cooking\n1,5,Sales,8,Cooking\n0,6,Person,0,Ghosts\n0,6,SoftwareEngineering,1,Ghosts\n0,6,Founder,2,Ghosts\n0,6,Bioinformatics,3,Ghosts\n0,6,Interest,4,Ghosts\n0,6,Exotic,5,Ghosts\n0,6,Supernatural,6,Ghosts\n0,6,SleepDisturber,7,Ghosts\n0,6,Sales,8,Ghosts\n8,4,Person,0,Bio\n8,4,SoftwareEngineering,1,Bio\n8,4,Founder,2,Bio\n8,4,Bioinformatics,3,Bio\n8,4,Interest,4,Bio\n8,4,Exotic,5,Bio\n8,4,Supernatural,6,Bio\n8,4,SleepDisturber,7,Bio\n8,4,Sales,8,Bio\n8,7,Person,0,Padel\n8,7,SoftwareEngineering,1,Padel\n8,7,Founder,2,Padel\n8,7,Bioinformatics,3,Padel\n8,7,Interest,4,Padel\n8,7,Exotic,5,Padel\n8,7,Supernatural,6,Padel\n8,7,SleepDisturber,7,Padel\n8,7,Sales,8,Padel\n9,2,Person,0,Computers\n9,2,SoftwareEngineering,1,Computers\n9,2,Founder,2,Computers\n9,2,Bioinformatics,3,Computers\n9,2,Interest,4,Computers\n9,2,Exotic,5,Computers\n9,2,Supernatural,6,Computers\n9,2,SleepDisturber,7,Computers\n9,2,Sales,8,Computers\n9,10,Person,0,Animals\n9,10,SoftwareEngineering,1,Animals\n9,10,Founder,2,Animals\n9,10,Bioinformatics,3,Animals\n9,10,Interest,4,Animals\n9,10,Exotic,5,Animals\n9,10,Supernatural,6,Animals\n9,10,SleepDisturber,7,Animals\n9,10,Sales,8,Animals\n11,5,Person,0,Cooking\n11,5,SoftwareEngineering,1,Cooking\n11,5,Founder,2,Cooking\n11,5,Bioinformatics,3,Cooking\n11,5,Interest,4,Cooking\n11,5,Exotic,5,Cooking\n11,5,Supernatural,6,Cooking\n11,5,SleepDisturber,7,Cooking\n11,5,Sales,8,Cooking\n15,13,Person,0,Gym\n15,13,SoftwareEngineering,1,Gym\n15,13,Founder,2,Gym\n15,13,Bioinformatics,3,Gym\n15,13,Interest,4,Gym\n15,13,Exotic,5,Gym\n15,13,Supernatural,6,Gym\n15,13,SleepDisturber,7,Gym\n15,13,Sales,8,Gym\n15,14,Person,0,Travel\n15,14,SoftwareEngineering,1,Travel\n15,14,Founder,2,Travel\n15,14,Bioinformatics,3,Travel\n15,14,Interest,4,Travel\n15,14,Exotic,5,Travel\n15,14,Supernatural,6,Travel\n15,14,SleepDisturber,7,Travel\n15,14,Sales,8,Travel\n12,13,Person,0,Gym\n12,13,SoftwareEngineering,1,Gym\n12,13,Founder,2,Gym\n12,13,Bioinformatics,3,Gym\n12,13,Interest,4,Gym\n12,13,Exotic,5,Gym\n12,13,Supernatural,6,Gym\n12,13,SleepDisturber,7,Gym\n12,13,Sales,8,Gym\n17,13,Person,0,Gym\n17,13,SoftwareEngineering,1,Gym\n17,13,Founder,2,Gym\n17,13,Bioinformatics,3,Gym\n17,13,Interest,4,Gym\n17,13,Exotic,5,Gym\n17,13,Supernatural,6,Gym\n17,13,SleepDisturber,7,Gym\n17,13,Sales,8,Gym\n12,16,Person,0,JiuJitsu\n12,16,SoftwareEngineering,1,JiuJitsu\n12,16,Founder,2,JiuJitsu\n12,16,Bioinformatics,3,JiuJitsu\n12,16,Interest,4,JiuJitsu\n12,16,Exotic,5,JiuJitsu\n12,16,Supernatural,6,JiuJitsu\n12,16,SleepDisturber,7,JiuJitsu\n12,16,Sales,8,JiuJitsu", + "resultJson": "{\"header\":{\"column_names\":[\"n\",\"m\",\"label\",\"id\",\"m.name\"],\"column_types\":[\"UInt64\",\"UInt64\",\"String\",\"UInt64\",\"String\"]},\"data\":[[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,11,11,11,11,11,11,11,11,11,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,12,12,12,12,12,12,12,12,12,17,17,17,17,17,17,17,17,17,12,12,12,12,12,12,12,12,12],[2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,4,4,4,4,4,4,4,4,4,7,7,7,7,7,7,7,7,7,2,2,2,2,2,2,2,2,2,10,10,10,10,10,10,10,10,10,5,5,5,5,5,5,5,5,5,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,16,16,16,16,16,16,16,16,16],[\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\",\"Person\",\"SoftwareEngineering\",\"Founder\",\"Bioinformatics\",\"Interest\",\"Exotic\",\"Supernatural\",\"SleepDisturber\",\"Sales\"],[0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8,0,1,2,3,4,5,6,7,8],[\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Eighties\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Ghosts\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Bio\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Padel\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Computers\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Animals\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Cooking\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Travel\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"Gym\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\",\"JiuJitsu\"]]]}" }, "tags": [ "match", diff --git a/test/query/ir/match/CMakeLists.txt b/test/query/ir/match/CMakeLists.txt index 1e98b3744f..4fd74e3c92 100644 --- a/test/query/ir/match/CMakeLists.txt +++ b/test/query/ir/match/CMakeLists.txt @@ -75,3 +75,5 @@ target_include_directories(test_query_ir_component_name_resolution PRIVATE ${IR_ add_call_v3_test(test_query_ir_double_dash DoubleDashTest.cpp) add_call_v3_test(test_query_ir_both_arrow_heads BothArrowHeadsTest.cpp) + +add_call_v3_test(test_query_ir_scan_edges_by_endpoint_labels ScanEdgesByEndpointLabelsTest.cpp) diff --git a/test/query/ir/match/ScanEdgesByEndpointLabelsTest.cpp b/test/query/ir/match/ScanEdgesByEndpointLabelsTest.cpp new file mode 100644 index 0000000000..89974b28ab --- /dev/null +++ b/test/query/ir/match/ScanEdgesByEndpointLabelsTest.cpp @@ -0,0 +1,195 @@ +#include + +#include +#include +#include + +#include "CallV3Test.h" +#include "StringRowSink.h" + +using namespace db; +using namespace turing::test; + +class ScanEdgesByEndpointLabelsTest : public CallV3Test { +protected: + void explain(std::string_view query, bool write, std::string& program) { + const std::string explained = std::string("EXPLAIN (db) ") + std::string(query); + + StringRowSink sink; + if (write) { + runWrite(explained, sink); + } else { + runQuery(explained, sink); + } + + for (const StringRowSink::Row& row : sink.getRows()) { + if (row.front() == "db") { + program = row.back(); + } + } + } + + void expectScan(std::string_view query, std::string_view scan, bool write = false) { + std::string program; + explain(query, write, program); + + EXPECT_NE(program.find(scan), std::string::npos) << program; + EXPECT_EQ(program.find("db.check_label_constraint"), std::string::npos) << program; + } + + void expectRows(std::string_view query, const std::vector& expected) { + StringRowSink sink; + runQuery(query, sink); + + std::vector rows; + sink.sortedRows(rows); + + EXPECT_EQ(rows, expected) << query; + } + + void expectWrittenRows(std::string_view query, const std::vector& expected) { + StringRowSink sink; + runWrite(query, sink); + + std::vector rows; + sink.sortedRows(rows); + + EXPECT_EQ(rows, expected) << query; + } +}; + +// Both ends carry labels, so the scan walks the index holding fewer edges: 7 Founder out-edges +// against 15 Interest in-edges, and 2 Exotic in-edges against 10 SoftwareEngineering out-edges. +TEST_F(ScanEdgesByEndpointLabelsTest, outHopReadsTheSourceIndexWhenTheSourceHasFewerNodes) { + const std::string_view query = "MATCH (n:Founder)-->(m:Interest) RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Founder\"], [\"Interest\"])"); + expectRows(query, + {{"Adam", "Bio"}, + {"Adam", "Cooking"}, + {"Remy", "Computers"}, + {"Remy", "Eighties"}, + {"Remy", "Ghosts"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, outHopReadsTheTargetIndexWhenTheTargetHasFewerNodes) { + const std::string_view query = "MATCH (n:SoftwareEngineering)-->(m:Exotic) RETURN n.name, m.name"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"SoftwareEngineering\"], [\"Exotic\"])"); + expectRows(query, {{"Remy", "Eighties"}, {"Remy", "Ghosts"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, inHopReadsTheTargetIndexWhenTheTargetHasFewerNodes) { + const std::string_view query = "MATCH (n:Exotic)<--(m:Person) RETURN n.name, m.name"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"Person\"], [\"Exotic\"])"); + expectRows(query, {{"Eighties", "Remy"}, {"Ghosts", "Remy"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, inHopReadsTheSourceIndexWhenTheSourceHasFewerNodes) { + const std::string_view query = "MATCH (n:Interest)<--(m:Founder) RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Founder\"], [\"Interest\"])"); + expectRows(query, + {{"Bio", "Adam"}, + {"Computers", "Remy"}, + {"Cooking", "Adam"}, + {"Eighties", "Remy"}, + {"Ghosts", "Remy"}}); +} + +// The pattern labels the target, but the 6 out-edges of the Bioinformatics nodes the WHERE +// labels the source by are fewer than the 15 Interest in-edges. +TEST_F(ScanEdgesByEndpointLabelsTest, whereOnTheSourceOfATargetLabelledScanMovesToTheSourceIndex) { + const std::string_view query = "MATCH (n)-->(m:Interest) WHERE n:Bioinformatics RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Bioinformatics\"], [\"Interest\"])"); + expectRows(query, + {{"Adam", "Bio"}, + {"Adam", "Cooking"}, + {"Martina", "Cooking"}, + {"Maxime", "Bio"}, + {"Maxime", "Padel"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, whereOnTheTargetOfASourceLabelledInHopKeepsTheSourceIndex) { + const std::string_view query = "MATCH (n)<--(m:Exotic) WHERE n:Person RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Exotic\"], [\"Person\"])"); + expectRows(query, {{"Remy", "Ghosts"}}); +} + +// The WHERE label lands on the end the scan already reads by its labels. +TEST_F(ScanEdgesByEndpointLabelsTest, whereOnTheLabelledSourceWidensItsLabels) { + const std::string_view query = "MATCH (n:Person)-->(m:Interest) WHERE n:Founder RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Person\", \"Founder\"], [\"Interest\"])"); + expectRows(query, + {{"Adam", "Bio"}, + {"Adam", "Cooking"}, + {"Remy", "Computers"}, + {"Remy", "Eighties"}, + {"Remy", "Ghosts"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, whereOnBothEndsOfASourceLabelledScan) { + const std::string_view query = "MATCH (n:Person)-->(m) WHERE n:Founder AND m:Exotic RETURN n.name, m.name"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"Person\", \"Founder\"], [\"Exotic\"])"); + expectRows(query, {{"Remy", "Eighties"}, {"Remy", "Ghosts"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, countsTheEdgesBetweenTheLabels) { + const std::string_view query = "MATCH (n:Person)-->(m:Interest) RETURN count(*)"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"Person\"], [\"Interest\"])"); + expectRows(query, {{"15"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, hopsOnFromTheLabelledTarget) { + const std::string_view query = "MATCH (n:Founder)-->(m:Exotic)-->(k) RETURN n.name, m.name, k.name"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"Founder\"], [\"Exotic\"])"); + expectRows(query, {{"Remy", "Ghosts", "Remy"}}); +} + +// A disjunction of labels is no label set an index holds, so it stays a check. +TEST_F(ScanEdgesByEndpointLabelsTest, disjunctionOnTheTargetStaysACheck) { + const std::string_view query = "MATCH (n:Person)-->(m:Interest) WHERE m:Exotic OR m:SleepDisturber RETURN n.name, m.name"; + + std::string program; + explain(query, false, program); + EXPECT_NE(program.find("db.scan_out_edges_by_label_src(["), std::string::npos) << program; + EXPECT_NE(program.find("db.check_label_constraint"), std::string::npos) << program; + + expectRows(query, {{"Luc", "Animals"}, {"Remy", "Eighties"}, {"Remy", "Ghosts"}}); +} + +// Bo is no Founder, so of the edges the CREATE wrote Ana's two are kept beside the five +// committed ones. +TEST_F(ScanEdgesByEndpointLabelsTest, keepsTheCreatedEdgesBetweenTheLabelsOffTheSourceIndex) { + const std::string_view query = "CREATE (a:Person:Founder {name: 'Ana'})-[:INTERESTED_IN]->(o:Interest:Exotic {name: 'Opera'}), " + "(a)-[:INTERESTED_IN]->(:Interest {name: 'Chess'}), " + "(:Person {name: 'Bo'})-[:INTERESTED_IN]->(o) " + "WITH count(*) AS created MATCH (n:Founder)-->(m:Interest) RETURN n.name, m.name"; + + expectScan(query, "db.scan_out_edges_by_label_src_tgt([\"Founder\"], [\"Interest\"])", true); + expectWrittenRows(query, + {{"Adam", "Bio"}, + {"Adam", "Cooking"}, + {"Ana", "Chess"}, + {"Ana", "Opera"}, + {"Remy", "Computers"}, + {"Remy", "Eighties"}, + {"Remy", "Ghosts"}}); +} + +TEST_F(ScanEdgesByEndpointLabelsTest, keepsTheCreatedEdgesBetweenTheLabelsOffTheTargetIndex) { + const std::string_view query = "CREATE (a:Person:Founder {name: 'Ana'})-[:INTERESTED_IN]->(o:Interest:Exotic {name: 'Opera'}), " + "(a)-[:INTERESTED_IN]->(:Interest {name: 'Chess'}), " + "(:Person {name: 'Bo'})-[:INTERESTED_IN]->(o) " + "WITH count(*) AS created MATCH (m:Exotic)<--(n:Person) RETURN n.name, m.name"; + + expectScan(query, "db.scan_in_edges_by_label_src_tgt([\"Person\"], [\"Exotic\"])", true); + expectWrittenRows(query, {{"Ana", "Opera"}, {"Bo", "Opera"}, {"Remy", "Eighties"}, {"Remy", "Ghosts"}}); +} diff --git a/test/query/ir/passes/CMakeLists.txt b/test/query/ir/passes/CMakeLists.txt index d9a46dc668..4b7eeb4819 100644 --- a/test/query/ir/passes/CMakeLists.txt +++ b/test/query/ir/passes/CMakeLists.txt @@ -48,6 +48,8 @@ add_ir_tests(test_query_ir_fuse_scan_edges_by_endpoint_label FuseScanEdgesByEndp add_turing_test(test_query_ir_fuse_scan_edges_by_endpoint_label_codegen FuseScanEdgesByEndpointLabelCodegenTest.cpp) target_include_directories(test_query_ir_fuse_scan_edges_by_endpoint_label_codegen PRIVATE ${IR_TEST_DIR}) +add_ir_tests(test_query_ir_edge_scan_index_cost EdgeScanIndexCostTest.cpp) + # The by-label hop fusion tests: the same two checks for the hops that walk the edges the # endpoint they reach carries a set of labels on. add_ir_tests(test_query_ir_fuse_edges_by_endpoint_label FuseEdgesByEndpointLabelTest.cpp) diff --git a/test/query/ir/passes/ConstantRowsPassesTest.cpp b/test/query/ir/passes/ConstantRowsPassesTest.cpp index 04d70c4242..674c2a3fda 100644 --- a/test/query/ir/passes/ConstantRowsPassesTest.cpp +++ b/test/query/ir/passes/ConstantRowsPassesTest.cpp @@ -125,7 +125,7 @@ TEST_F(ConstantRowsPassesTest, placesFactorsAndReRootsUnderAConstantProjection) const PassCase cases[] = { {"MATCH (a:Person), (b:Interest) WHERE b.name = 'Bio'", "db.scan_nodes_by_property_value", 8}, {"MATCH (a)-->(b:Person {name: 'Remy'})", "db.scan_out_edges_by_label_tgt", 2}, - {"MATCH (a:Interest)<--(b:Person {name: 'Adam'})", "db.scan_in_edges_by_label_tgt", 2}, + {"MATCH (a:Interest)<--(b:Person {name: 'Adam'})", "db.scan_in_edges_by_label_src_tgt", 2}, }; for (const PassCase& passCase : cases) { diff --git a/test/query/ir/passes/EdgeScanIndexCostTest.cpp b/test/query/ir/passes/EdgeScanIndexCostTest.cpp new file mode 100644 index 0000000000..a608a18ef4 --- /dev/null +++ b/test/query/ir/passes/EdgeScanIndexCostTest.cpp @@ -0,0 +1,154 @@ +#include + +#include + +#include +#include +#include +#include + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/MLIRContext.h" +#include "llvm/Support/raw_ostream.h" + +#include "Graph.h" +#include "JobSystem.h" +#include "metadata/LabelSet.h" +#include "reader/GraphReader.h" +#include "versioning/Change.h" +#include "versioning/CommitBuilder.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" +#include "writers/DataPartBuilder.h" +#include "writers/MetadataBuilder.h" + +#include "CypherAnalyzer.h" +#include "CypherAST.h" +#include "CypherParser.h" + +#include "DBDialect.h" +#include "DBPasses.h" +#include "DBProgramGenerator.h" +#include "NLDialect.h" +#include "StorageDialect.h" + +#include "TuringTest.h" + +using namespace db; +using namespace turing::test; + +// Which index a scan labelled on both ends walks. The walk visits every edge the index holds +// under that end's labels, so the end with fewer edges is the cheaper one, whatever its node +// count. Here 2 Hub nodes hold 105 out-edges and 5 Target nodes 5 in-edges. +class EdgeScanIndexCostTest : public TuringTest { +protected: + void initialize() override { + _jobSystem = std::make_unique(); + _jobSystem->init(); + + buildGraph(); + } + + void terminate() override { + _jobSystem->terminate(); + } + + // 2 Hub nodes, 100 Leaf nodes and 5 Target nodes. The first hub links to every Leaf and + // Target node, the second to none, and every Target node links to one Leaf node. + void buildGraph() { + _graph = Graph::create(); + + auto change = _graph->newChange(); + auto* commitBuilder = change->access().getTip(); + auto& builder = commitBuilder->newBuilder(); + auto& metadata = builder.getMetadata(); + + const LabelSet hubLabelSet = LabelSet::fromList({metadata.getOrCreateLabel("Hub")}); + const LabelSet leafLabelSet = LabelSet::fromList({metadata.getOrCreateLabel("Leaf")}); + const LabelSet targetLabelSet = LabelSet::fromList({metadata.getOrCreateLabel("Target")}); + const EdgeTypeID linksType = metadata.getOrCreateEdgeType("LINKS"); + + const NodeID hub = builder.addNode(hubLabelSet); + builder.addNode(hubLabelSet); + + std::vector leaves; + for (size_t leaf = 0; leaf < 100; leaf++) { + leaves.push_back(builder.addNode(leafLabelSet)); + builder.addEdge(linksType, hub, leaves.back()); + } + + for (size_t target = 0; target < 5; target++) { + const NodeID targetNode = builder.addNode(targetLabelSet); + builder.addEdge(linksType, hub, targetNode); + builder.addEdge(linksType, targetNode, leaves[target]); + } + + const auto submitResult = change->access().submit(*_jobSystem); + ASSERT_TRUE(submitResult); + } + + // The db program the pipeline leaves for a query on this graph, as text. + void generate(std::string_view query, std::string& program) { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + const GraphView view = reader.getView(); + const mlir::db::DBPassContext context {._view = &view}; + + CypherAST ast(nullptr, query); + CypherParser parser(&ast); + parser.parse(query); + + CypherAnalyzer analyzer(&ast, view); + analyzer.analyze(); + + mlir::MLIRContext mlirContext; + mlirContext.getOrLoadDialect(); + mlirContext.getOrLoadDialect(); + mlirContext.getOrLoadDialect(); + mlirContext.getOrLoadDialect(); + + mlir::OpBuilder builder(&mlirContext); + mlir::OwningOpRef owningModule = mlir::ModuleOp::create(builder.getUnknownLoc()); + mlir::ModuleOp module = owningModule.get(); + + DBProgramGenerator generator(&module, nullptr, context); + generator.generate(&ast); + + program.clear(); + llvm::raw_string_ostream stream(program); + module.print(stream); + } + + void expectScan(std::string_view query, std::string_view scan) { + std::string program; + generate(query, program); + + EXPECT_NE(program.find(scan), std::string::npos) << program; + } + + std::unique_ptr _jobSystem; + std::unique_ptr _graph; +}; + +TEST_F(EdgeScanIndexCostTest, outHopWalksTheTargetIndexWhenItHoldsFewerEdges) { + expectScan("MATCH (h:Hub)-->(t:Target) RETURN h, t", + "db.scan_in_edges_by_label_src_tgt([\"Hub\"], [\"Target\"])"); +} + +TEST_F(EdgeScanIndexCostTest, inHopWalksTheTargetIndexWhenItHoldsFewerEdges) { + expectScan("MATCH (t:Target)<--(h:Hub) RETURN h, t", + "db.scan_in_edges_by_label_src_tgt([\"Hub\"], [\"Target\"])"); +} + +// 5 Target out-edges against 105 Leaf in-edges: the source index, which node counts pick too. +TEST_F(EdgeScanIndexCostTest, outHopWalksTheSourceIndexWhenItHoldsFewerEdges) { + expectScan("MATCH (t:Target)-->(l:Leaf) RETURN t, l", + "db.scan_out_edges_by_label_src_tgt([\"Target\"], [\"Leaf\"])"); +} + +// 5 Target out-edges against 5 Target in-edges: a tie reads the source's index. +TEST_F(EdgeScanIndexCostTest, tieWalksTheSourceIndex) { + expectScan("MATCH (a:Target)<--(b:Target) RETURN a, b", + "db.scan_out_edges_by_label_src_tgt([\"Target\"], [\"Target\"])"); +} diff --git a/test/query/ir/passes/FuseScanEdgesByEndpointLabelCodegenTest.cpp b/test/query/ir/passes/FuseScanEdgesByEndpointLabelCodegenTest.cpp index eddc93460b..ba179d6fe0 100644 --- a/test/query/ir/passes/FuseScanEdgesByEndpointLabelCodegenTest.cpp +++ b/test/query/ir/passes/FuseScanEdgesByEndpointLabelCodegenTest.cpp @@ -170,14 +170,14 @@ TEST_F(FuseScanEdgesByEndpointLabelCodegenTest, labelledTargetOfAnInHopKeepsItsO EXPECT_EQ(countOps(*module), 0u); } -// Both ends carry labels: the hop's own end fuses into the by-label scan and the other -// stays a label check over it, since no op holds a label set per endpoint. -TEST_F(FuseScanEdgesByEndpointLabelCodegenTest, labelledBothEndsKeepsTheCheckOnTheFarOne) { +// Both ends carry labels: the hop's own end fuses into the by-label scan, and the check on +// the other end then folds into it as well. +TEST_F(FuseScanEdgesByEndpointLabelCodegenTest, labelledBothEndsFuseIntoOneScan) { const mlir::OwningOpRef module = generate("MATCH (a:Person)-->(b:Interest) RETURN a, b"); - EXPECT_EQ(countOps(*module), 1u); - EXPECT_EQ(countOps(*module), 0u); - EXPECT_EQ(countOps(*module), 1u); + EXPECT_EQ(countOps(*module), 1u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); } // An undirected hop reads both edge directions of its nodes, which is no edge scan. diff --git a/test/query/ir/passes/FuseScanEdgesByEndpointLabelTest.cpp b/test/query/ir/passes/FuseScanEdgesByEndpointLabelTest.cpp index 53452e9c96..5aaa9de969 100644 --- a/test/query/ir/passes/FuseScanEdgesByEndpointLabelTest.cpp +++ b/test/query/ir/passes/FuseScanEdgesByEndpointLabelTest.cpp @@ -112,6 +112,92 @@ func.func @main() { } )mlir"; +// MATCH (a:Person)-->(b:Interest) RETURN a, b +const char* const sourceLabelledScanWithLabelledTarget = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src(["Person"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%tgts) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Interest"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + +// MATCH (a)-->(b:Interest) WHERE a:Bioinformatics RETURN a, b +const char* const targetLabelledScanWithLabelledSource = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_tgt(["Interest"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%srcs) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Bioinformatics"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + +// MATCH (a:Person)-->(b) WHERE a:Founder RETURN a, b +const char* const sourceLabelledScanWithSourceCheck = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src(["Person"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%srcs) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Founder"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + +// MATCH (a:Person)-->(b:Interest) WHERE b:Exotic RETURN a, b +const char* const bothLabelledScanWithTargetCheck = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src_tgt(["Person"], ["Interest"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%tgts) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Exotic"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + +const char* const sourceLabelledScanWithAbsentLabelledTarget = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src(["Person"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%tgts) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Wizard"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + +// MATCH (a)-->(b) WHERE b:Interest AND a:Bioinformatics RETURN a, b +const char* const edgeScanWithStackedEndpointChecks = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_edges() : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%tgts) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, ["Interest"]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + %labelsets2 = db.get_node_label_set(%kept#0) : (!db.column) -> !db.column + %matches2 = db.check_label_constraint(%labelsets2, ["Bioinformatics"]) : (!db.column) -> !db.column + %kept2:2 = db.filter(%matches2, {%kept#0, %kept#1}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept2#0, %kept2#1) : !db.column, !db.column + return +} +)mlir"; + +// A disjunction is no label set an index holds. +const char* const sourceLabelledScanWithTargetDisjunction = R"mlir( +func.func @main() { + %srcs, %eids, %etypes, %tgts = db.scan_out_edges_by_label_src(["Person"]) : !db.column, !db.column, !db.column, !db.column + %labelsets = db.get_node_label_set(%tgts) : (!db.column) -> !db.column + %matches = db.check_label_constraint(%labelsets, [["Exotic"], ["SleepDisturber"]]) : (!db.column) -> !db.column + %kept:2 = db.filter(%matches, {%srcs, %tgts}) : (!db.column, !db.column, !db.column) -> (!db.column, !db.column) + db.output(%kept#0, %kept#1) : !db.column, !db.column + return +} +)mlir"; + // The fused forms spelled by hand, which is what the pass prints. const char* const labelledTargetEdgeScan = R"mlir( func.func @main() { @@ -236,7 +322,7 @@ class FuseScanEdgesByEndpointLabelTest : public TuringTest { CollectingPairSink sink; LocalMemory memory; - NLInterpreter interpreter(*nlModule, &view, &sink, &memory, ChunkConfig::CHUNK_SIZE); + NLInterpreter interpreter(*nlModule, &view, &sink, &memory, _chunkSize); interpreter.run(); sink.sortedPairs(pairs); @@ -265,6 +351,7 @@ class FuseScanEdgesByEndpointLabelTest : public TuringTest { } mlir::MLIRContext _context; + size_t _chunkSize {ChunkConfig::CHUNK_SIZE}; }; TEST_F(FuseScanEdgesByEndpointLabelTest, fusesEdgeScanAndTargetLabelCheck) { @@ -462,3 +549,157 @@ TEST_F(FuseScanEdgesByEndpointLabelTest, emitsTheSameEdgesAsTheLabelledHops) { EXPECT_FALSE(sourcePairs.empty()); EXPECT_EQ(sourcePairs, checkedSourcePairs); } + +TEST_F(FuseScanEdgesByEndpointLabelTest, fusesSourceLabelledScanAndTargetLabelCheck) { + const mlir::OwningOpRef module = parse(sourceLabelledScanWithLabelledTarget); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + llvm::SmallVector edgeScans = collect(*module); + ASSERT_EQ(edgeScans.size(), 1u); + mlir::db::ScanOutEdgesByLabelSrcTgt edgeScan = edgeScans.front(); + + ASSERT_EQ(edgeScan.getSrcLabels().size(), 1u); + EXPECT_EQ(mlir::cast(edgeScan.getSrcLabels()[0]).getValue(), "Person"); + ASSERT_EQ(edgeScan.getTgtLabels().size(), 1u); + EXPECT_EQ(mlir::cast(edgeScan.getTgtLabels()[0]).getValue(), "Interest"); + + llvm::SmallVector outputs = collect(*module); + ASSERT_EQ(outputs.size(), 1u); + const mlir::Operation::operand_range columns = outputs.front().getColumns(); + ASSERT_EQ(columns.size(), 2u); + EXPECT_EQ(columns[0], edgeScan.getSrcids()); + EXPECT_EQ(columns[1], edgeScan.getTgtids()); + + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); +} + +// The scan keeps the index it read, which is keyed by the target's label set. +TEST_F(FuseScanEdgesByEndpointLabelTest, fusesTargetLabelledScanAndSourceLabelCheck) { + const mlir::OwningOpRef module = parse(targetLabelledScanWithLabelledSource); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + llvm::SmallVector edgeScans = collect(*module); + ASSERT_EQ(edgeScans.size(), 1u); + mlir::db::ScanInEdgesByLabelSrcTgt edgeScan = edgeScans.front(); + + ASSERT_EQ(edgeScan.getSrcLabels().size(), 1u); + EXPECT_EQ(mlir::cast(edgeScan.getSrcLabels()[0]).getValue(), "Bioinformatics"); + ASSERT_EQ(edgeScan.getTgtLabels().size(), 1u); + EXPECT_EQ(mlir::cast(edgeScan.getTgtLabels()[0]).getValue(), "Interest"); + + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, widensTheLabelsOfTheCheckedEnd) { + const mlir::OwningOpRef module = parse(sourceLabelledScanWithSourceCheck); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + llvm::SmallVector edgeScans = collect(*module); + ASSERT_EQ(edgeScans.size(), 1u); + + const mlir::ArrayAttr labels = edgeScans.front().getLabels(); + ASSERT_EQ(labels.size(), 2u); + EXPECT_EQ(mlir::cast(labels[0]).getValue(), "Person"); + EXPECT_EQ(mlir::cast(labels[1]).getValue(), "Founder"); + + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, widensTheTargetLabelsOfABothEndsScan) { + const mlir::OwningOpRef module = parse(bothLabelledScanWithTargetCheck); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + llvm::SmallVector edgeScans = collect(*module); + ASSERT_EQ(edgeScans.size(), 1u); + mlir::db::ScanOutEdgesByLabelSrcTgt edgeScan = edgeScans.front(); + + EXPECT_EQ(edgeScan.getSrcLabels().size(), 1u); + + const mlir::ArrayAttr targetLabels = edgeScan.getTgtLabels(); + ASSERT_EQ(targetLabels.size(), 2u); + EXPECT_EQ(mlir::cast(targetLabels[0]).getValue(), "Interest"); + EXPECT_EQ(mlir::cast(targetLabels[1]).getValue(), "Exotic"); + + EXPECT_EQ(countOps(*module), 0u); +} + +// The first check makes a target-labelled scan, and the second then lands on its source. +TEST_F(FuseScanEdgesByEndpointLabelTest, fusesStackedChecksOnBothEnds) { + const mlir::OwningOpRef module = parse(edgeScanWithStackedEndpointChecks); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + EXPECT_EQ(countOps(*module), 1u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 0u); +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, leavesTargetDisjunctionAlone) { + const mlir::OwningOpRef module = parse(sourceLabelledScanWithTargetDisjunction); + ASSERT_TRUE(module); + ASSERT_TRUE(runFuse(*module)); + ASSERT_TRUE(mlir::succeeded(mlir::verify(*module))); + + EXPECT_EQ(countOps(*module), 1u); + EXPECT_EQ(countOps(*module), 0u); + EXPECT_EQ(countOps(*module), 1u); +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, emitsTheSameEdgesOffTheSourceIndex) { + for (const size_t chunkSize : {ChunkConfig::CHUNK_SIZE, size_t {1}, size_t {2}}) { + _chunkSize = chunkSize; + + std::vector> unfused; + std::vector> fused; + runPairsBeforeAndAfterFusion(sourceLabelledScanWithLabelledTarget, unfused, fused); + + EXPECT_EQ(unfused.size(), 15u) << chunkSize; + EXPECT_EQ(fused, unfused) << chunkSize; + } +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, emitsTheSameEdgesOffTheTargetIndex) { + for (const size_t chunkSize : {ChunkConfig::CHUNK_SIZE, size_t {1}, size_t {2}}) { + _chunkSize = chunkSize; + + std::vector> unfused; + std::vector> fused; + runPairsBeforeAndAfterFusion(targetLabelledScanWithLabelledSource, unfused, fused); + + EXPECT_EQ(unfused.size(), 5u) << chunkSize; + EXPECT_EQ(fused, unfused) << chunkSize; + } +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, emitsTheSameEdgesForStackedChecks) { + std::vector> unfused; + std::vector> fused; + runPairsBeforeAndAfterFusion(edgeScanWithStackedEndpointChecks, unfused, fused); + + EXPECT_EQ(unfused.size(), 5u); + EXPECT_EQ(fused, unfused); +} + +TEST_F(FuseScanEdgesByEndpointLabelTest, emitsNothingForAnAbsentLabelOnTheOtherEnd) { + std::vector> unfused; + std::vector> fused; + runPairsBeforeAndAfterFusion(sourceLabelledScanWithAbsentLabelledTarget, unfused, fused); + + EXPECT_TRUE(unfused.empty()); + EXPECT_TRUE(fused.empty()); +} diff --git a/test/storage/CMakeLists.txt b/test/storage/CMakeLists.txt index 40c3a10ff1..9f490d46e4 100644 --- a/test/storage/CMakeLists.txt +++ b/test/storage/CMakeLists.txt @@ -35,6 +35,9 @@ add_turing_test(test_storage_comparator ComparatorTest.cpp) add_turing_test(test_storage_arcmanager ArcManagerTest.cpp) add_turing_test(test_storage_graphwriter GraphWriterTest.cpp) add_turing_test(test_storage_node_count_with_property NodeCountWithPropertyTest.cpp) +add_turing_test(test_storage_node_count_with_property_exclusions NodeCountWithPropertyExclusionsTest.cpp) +add_turing_test(test_storage_edge_count_matching_labelset EdgeCountMatchingLabelsetTest.cpp) +add_turing_test(test_storage_node_count_matching_labelset NodeCountMatchingLabelsetTest.cpp) add_turing_test(test_storage_iterator iterators/IteratorsTest.cpp) add_turing_test(test_storage_scan_nodes_by_property_value iterators/ScanNodesByPropertyValueTest.cpp) @@ -48,6 +51,7 @@ add_turing_test(test_storage_get_edges_by_label_iterator iterators/GetEdgesByLab add_turing_test(test_storage_get_edges_by_type_and_label_iterator iterators/GetEdgesByTypeAndLabelIteratorTest.cpp) add_turing_test(test_storage_scan_edges_by_type_iterator iterators/ScanEdgesByTypeIteratorTest.cpp) add_turing_test(test_storage_scan_edges_by_label_tombstone iterators/ScanEdgesByLabelTombstoneTest.cpp) +add_turing_test(test_storage_scan_edges_by_endpoint_labels_chunk_writer iterators/ScanEdgesByEndpointLabelsChunkWriterTest.cpp) add_turing_test(test_storage_get_node_labelset_iterator iterators/GetNodeLabelSetIteratorTest.cpp) add_turing_test(test_storage_get_edge_types_iterator iterators/GetEdgeTypesIteratorTest.cpp) add_turing_test(test_storage_neighbourhood_sample_iterator iterators/NeighbourhoodSampleIteratorTest.cpp) diff --git a/test/storage/EdgeCountMatchingLabelsetTest.cpp b/test/storage/EdgeCountMatchingLabelsetTest.cpp new file mode 100644 index 0000000000..67df62e941 --- /dev/null +++ b/test/storage/EdgeCountMatchingLabelsetTest.cpp @@ -0,0 +1,116 @@ +#include + +#include + +#include +#include +#include + +#include "Graph.h" +#include "JobSystem.h" +#include "columns/ColumnEdgeTypes.h" +#include "columns/ColumnIDs.h" +#include "iterators/ChunkConfig.h" +#include "iterators/ScanInEdgesByTargetLabelIterator.h" +#include "iterators/ScanOutEdgesBySourceLabelIterator.h" +#include "metadata/GraphMetadata.h" +#include "metadata/LabelSet.h" +#include "metadata/LabelSetHandle.h" +#include "reader/GraphReader.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" +#include "writers/GraphWriter.h" + +#include "SimpleGraph.h" +#include "TuringTest.h" + +using namespace db; +using namespace turing::test; + +namespace { + +template +size_t countScannedEdges(const GraphView& view, const LabelSet& labelset) { + ChunkWriterType chunkWriter(view, LabelSetHandle(labelset)); + + ColumnNodeIDs sources; + ColumnEdgeIDs edgeIDs; + ColumnEdgeTypes edgeTypes; + ColumnNodeIDs targets; + chunkWriter.setSrcIDs(&sources); + chunkWriter.setEdgeIDs(&edgeIDs); + chunkWriter.setEdgeTypes(&edgeTypes); + chunkWriter.setTgtIDs(&targets); + + size_t count = 0; + while (chunkWriter.isValid()) { + chunkWriter.fill(ChunkConfig::CHUNK_SIZE); + count += edgeIDs.size(); + } + + return count; +} + +} + +// The edges a by-label edge index holds under a label set are the edges a scan of that index +// walks. A later part adds edges out of and into nodes of earlier ones, which its index +// holds as patch nodes, so the counts have to take those spans in as well. +class EdgeCountMatchingLabelsetTest : public TuringTest { +protected: + void initialize() override { + _jobSystem = std::make_unique(); + _jobSystem->init(); + + _graph = Graph::create(); + SimpleGraph::createSimpleGraph(_graph.get()); + + GraphWriter writer(_graph.get(), _jobSystem.get()); + const NodeID remy {0}; + const NodeID adam {1}; + const NodeID ghosts {6}; + const NodeID chess = writer.addNode({"Interest"}); + writer.addEdge("KNOWS_WELL", remy, adam); + writer.addEdge("INTERESTED_IN", adam, ghosts); + writer.addEdge("INTERESTED_IN", remy, chess); + writer.submit(); + } + + void terminate() override { + _jobSystem->terminate(); + } + + void expectCountsMatchTheScans(std::initializer_list names) { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + const GraphView view = reader.getView(); + ASSERT_GT(reader.getDatapartCount(), 1u); + + LabelSet labelset; + for (const std::string_view name : names) { + const std::optional label = reader.getMetadata().labels().get(name); + ASSERT_TRUE(label.has_value()); + labelset.set(*label); + } + + const LabelSetHandle handle(labelset); + EXPECT_EQ(reader.getOutEdgeCountMatchingLabelset(handle), + countScannedEdges(view, labelset)); + EXPECT_EQ(reader.getInEdgeCountMatchingLabelset(handle), + countScannedEdges(view, labelset)); + } + + std::unique_ptr _jobSystem; + std::unique_ptr _graph; +}; + +TEST_F(EdgeCountMatchingLabelsetTest, countsTheEdgesOfOneLabel) { + expectCountsMatchTheScans({"Person"}); + expectCountsMatchTheScans({"Interest"}); + expectCountsMatchTheScans({"Exotic"}); +} + +TEST_F(EdgeCountMatchingLabelsetTest, countsTheEdgesOfNodesCarryingEveryLabel) { + expectCountsMatchTheScans({"Person", "Founder"}); + expectCountsMatchTheScans({"Interest", "Exotic"}); +} diff --git a/test/storage/NodeCountMatchingLabelsetTest.cpp b/test/storage/NodeCountMatchingLabelsetTest.cpp new file mode 100644 index 0000000000..0a5cccb3d1 --- /dev/null +++ b/test/storage/NodeCountMatchingLabelsetTest.cpp @@ -0,0 +1,108 @@ +#include + +#include + +#include +#include +#include +#include + +#include "Graph.h" +#include "JobSystem.h" +#include "metadata/GraphMetadata.h" +#include "metadata/LabelSet.h" +#include "metadata/LabelSetHandle.h" +#include "reader/GraphReader.h" +#include "versioning/Tombstones.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" +#include "writers/GraphWriter.h" + +#include "SimpleGraph.h" +#include "TuringTest.h" + +using namespace db; +using namespace turing::test; + +// Deleted nodes are left out of the count of the nodes matching a label set. The 7 deleted +// Interest nodes are fewer than the 10 Interest nodes and more than the 2 Exotic ones, so +// the counts below take both ways of leaving them out: checking each matched node against +// the deletions, and checking each deletion against the labels. +class NodeCountMatchingLabelsetTest : public TuringTest { +protected: + void initialize() override { + _jobSystem = std::make_unique(); + _jobSystem->init(); + + _graph = Graph::create(); + SimpleGraph::createSimpleGraph(_graph.get()); + + const NodeID eighties {3}; + const NodeID bio {4}; + const NodeID cooking {5}; + const NodeID padel {7}; + const NodeID gym {13}; + const NodeID travel {14}; + const NodeID jiuJitsu {16}; + + { + GraphWriter writer(_graph.get(), _jobSystem.get()); + writer.deleteNodes({eighties, bio, cooking, padel}); + writer.submit(); + } + + { + GraphWriter writer(_graph.get(), _jobSystem.get()); + writer.deleteNodes({gym, travel, jiuJitsu}); + writer.submit(); + } + } + + void terminate() override { + _jobSystem->terminate(); + } + + void expectCount(std::initializer_list names, size_t expectedCount) { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + const GraphView view = reader.getView(); + const Tombstones& tombstones = view.tombstones(); + ASSERT_EQ(tombstones.numNodes(), 7u); + + LabelSet labelset; + for (const std::string_view name : names) { + const std::optional label = reader.getMetadata().labels().get(name); + ASSERT_TRUE(label.has_value()); + labelset.set(*label); + } + + const LabelSetHandle handle(labelset); + + size_t matchingNodes = 0; + for (size_t id = 0; id < reader.getTotalNodesAllocated(); id++) { + const NodeID nodeID {id}; + const LabelSetHandle nodeLabels = reader.getNodeLabelSet(nodeID); + if (nodeLabels.isValid() && nodeLabels.hasAtLeastLabels(handle) && !tombstones.containsNode(nodeID)) { + matchingNodes++; + } + } + + EXPECT_EQ(matchingNodes, expectedCount); + EXPECT_EQ(reader.getNodeCountMatchingLabelset(handle), expectedCount); + } + + std::unique_ptr _jobSystem; + std::unique_ptr _graph; +}; + +TEST_F(NodeCountMatchingLabelsetTest, leavesOutDeletedNodesFewerThanTheMatchedOnes) { + expectCount({"Interest"}, 3); + expectCount({"Person"}, 8); + expectCount({}, 11); +} + +TEST_F(NodeCountMatchingLabelsetTest, leavesOutDeletedNodesMoreThanTheMatchedOnes) { + expectCount({"Exotic"}, 1); + expectCount({"Interest", "SleepDisturber"}, 2); + expectCount({"Founder"}, 2); +} diff --git a/test/storage/NodeCountWithPropertyExclusionsTest.cpp b/test/storage/NodeCountWithPropertyExclusionsTest.cpp new file mode 100644 index 0000000000..353b15f1f0 --- /dev/null +++ b/test/storage/NodeCountWithPropertyExclusionsTest.cpp @@ -0,0 +1,147 @@ +#include + +#include +#include + +#include +#include +#include + +#include "Graph.h" +#include "JobSystem.h" +#include "metadata/LabelSet.h" +#include "metadata/LabelSetHandle.h" +#include "metadata/PropertyType.h" +#include "reader/GraphReader.h" +#include "versioning/Change.h" +#include "versioning/ChangeAccessor.h" +#include "versioning/CommitBuilder.h" +#include "versioning/Tombstones.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" +#include "writers/DataPartBuilder.h" +#include "writers/GraphWriter.h" +#include "writers/MetadataBuilder.h" + +#include "TuringTest.h" + +using namespace db; +using namespace turing::test; + +namespace { + +constexpr size_t FIRST_PART_NODE_COUNT = 40; +constexpr size_t RARE_NODE_COUNT = 4; + +} + +// A node's value for a property is its entry in the newest part holding one, and a deleted +// node has none. The first part holds 40 Row nodes, the first 4 of them Rare as well. A later +// part rewrites nodes 0 and 5 and nulls nodes 1 and 7, a later one deletes nodes 0 and 9, and +// the last adds 2 Row nodes. Node 0 is both rewritten and deleted, so it must be left out once. +// The 4 Rare entries of the first part are fewer than its 6 exclusions and the 40 Row entries +// more, so the two label sets take both ways of finding the excluded entries. +class NodeCountWithPropertyExclusionsTest : public TuringTest { +protected: + void initialize() override { + _jobSystem = std::make_unique(); + _jobSystem->init(); + _graph = Graph::create(); + + writePart(FIRST_PART_NODE_COUNT, {}, {}); + writePart(0, {{0, 111}, {5, 555}}, {1, 7}); + + GraphWriter writer(_graph.get(), _jobSystem.get()); + writer.deleteNodes({NodeID {0}, NodeID {9}}); + writer.submit(); + + writePart(2, {}, {}); + } + + void terminate() override { + _jobSystem->terminate(); + } + + void writePart(size_t newNodeCount, + const std::vector>& rewrites, + const std::vector& nulls) { + std::unique_ptr change = _graph->newChange(); + CommitBuilder* commit = change->access().getTip(); + DataPartBuilder& builder = commit->newBuilder(); + MetadataBuilder& metadata = builder.getMetadata(); + + _rowLabel = metadata.getOrCreateLabel("Row"); + _rareLabel = metadata.getOrCreateLabel("Rare"); + _valueID = metadata.getOrCreatePropertyType("value", ValueType::Int64)._id; + + for (size_t node = 0; node < newNodeCount; node++) { + LabelSet labelset; + labelset.set(_rowLabel); + if (_nodeCount < RARE_NODE_COUNT) { + labelset.set(_rareLabel); + } + + const NodeID nodeID = builder.addNode(labelset); + builder.addNodeProperty(nodeID, _valueID, static_cast(_nodeCount) * 10); + _nodeCount++; + } + + for (const auto& [node, value] : rewrites) { + builder.addNodeProperty(NodeID {node}, _valueID, value); + } + + for (const uint64_t node : nulls) { + builder.addNodeProperty(NodeID {node}, _valueID, std::nullopt); + } + + ASSERT_TRUE(change->access().submit(*_jobSystem)); + } + + // The live nodes of the label set the per-node lookup resolves a value for. + size_t expectedCount(const GraphReader& reader, const LabelSetHandle& labelset) const { + const Tombstones& tombstones = reader.getView().tombstones(); + + size_t count = 0; + for (uint64_t node = 0; node < _nodeCount; node++) { + const NodeID nodeID {node}; + if (tombstones.containsNode(nodeID) || !reader.getNodeLabelSet(nodeID).hasAtLeastLabels(labelset)) { + continue; + } + + const std::optional value = reader.tryGetNodeProperty(_valueID, nodeID); + if (value.has_value() && value.value()) { + count++; + } + } + + return count; + } + + void expectCount(LabelID label, size_t count) { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + + const LabelSet labelset = LabelSet::fromList({label}); + const LabelSetHandle handle(labelset); + + EXPECT_EQ(expectedCount(reader, handle), count); + EXPECT_EQ(reader.getNodeCountWithProperty(handle, _valueID), count); + } + + std::unique_ptr _jobSystem; + std::unique_ptr _graph; + size_t _nodeCount {0}; + PropertyTypeID _valueID; + LabelID _rowLabel; + LabelID _rareLabel; +}; + +// 42 Row nodes, less the 2 deleted and the 2 nulled. +TEST_F(NodeCountWithPropertyExclusionsTest, leavesOutExclusionsFewerThanTheMatchedEntries) { + expectCount(_rowLabel, 38); +} + +// Of the 4 Rare nodes, node 0 is deleted and node 1 nulled. +TEST_F(NodeCountWithPropertyExclusionsTest, leavesOutExclusionsMoreThanTheMatchedEntries) { + expectCount(_rareLabel, 2); +} diff --git a/test/storage/iterators/ScanEdgesByEndpointLabelsChunkWriterTest.cpp b/test/storage/iterators/ScanEdgesByEndpointLabelsChunkWriterTest.cpp new file mode 100644 index 0000000000..eb39ad1d6d --- /dev/null +++ b/test/storage/iterators/ScanEdgesByEndpointLabelsChunkWriterTest.cpp @@ -0,0 +1,237 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "TuringTest.h" + +#include "Graph.h" +#include "columns/ColumnEdgeTypes.h" +#include "columns/ColumnIDs.h" +#include "iterators/ChunkConfig.h" +#include "iterators/ScanEdgesByEndpointLabelsChunkWriter.h" +#include "iterators/ScanInEdgesByTargetLabelIterator.h" +#include "iterators/ScanOutEdgesBySourceLabelIterator.h" +#include "metadata/GraphMetadata.h" +#include "metadata/LabelSet.h" +#include "reader/GraphReader.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" +#include "writers/GraphWriter.h" + +#include "JobSystem.h" +#include "SimpleGraph.h" + +using namespace db; +using namespace turing::test; + +namespace { + +// One row as the writer lays it out: the indexed end, the edge, its type and the other end. +using ScannedRow = std::tuple; + +struct ScannedChunk { + ColumnNodeIDs _indexedEnds; + ColumnEdgeIDs _edgeIDs; + ColumnEdgeTypes _edgeTypes; + ColumnNodeIDs _otherEnds; +}; + +template +void bindChunk(ChunkWriterType& chunkWriter, ScannedChunk& chunk) { + chunkWriter.setSrcIDs(&chunk._indexedEnds); + chunkWriter.setEdgeIDs(&chunk._edgeIDs); + chunkWriter.setEdgeTypes(&chunk._edgeTypes); + chunkWriter.setTgtIDs(&chunk._otherEnds); +} + +void appendRows(const ScannedChunk& chunk, std::vector& rows) { + for (size_t row = 0; row < chunk._edgeIDs.size(); row++) { + rows.emplace_back(chunk._indexedEnds[row].getValue(), + chunk._edgeIDs[row].getValue(), + chunk._edgeTypes[row].getValue(), + chunk._otherEnds[row].getValue()); + } +} + +// Drives the writer as the executor does, one fill of chunkSize rows at a time, and checks +// that no fill writes more than chunkSize rows and every fill leaves the four columns aligned. +template +void scanInChunks(ChunkWriterType& chunkWriter, size_t chunkSize, std::vector& rows) { + ScannedChunk chunk; + bindChunk(chunkWriter, chunk); + + while (chunkWriter.isValid()) { + chunkWriter.fill(chunkSize); + + const size_t rowCount = chunk._edgeIDs.size(); + ASSERT_LE(rowCount, chunkSize); + ASSERT_EQ(chunk._indexedEnds.size(), rowCount); + ASSERT_EQ(chunk._edgeTypes.size(), rowCount); + ASSERT_EQ(chunk._otherEnds.size(), rowCount); + + appendRows(chunk, rows); + } + + std::sort(rows.begin(), rows.end()); +} + +} + +// The both-labels writer keeps the edges of one end's index whose other end carries a second +// label set. Its rows are checked against the single-end writer over the same index, with the +// other end filtered here, so a range cut short by the chunk size or a row dropped by a +// tombstone shows up as a missing, extra or misaligned row. +class ScanEdgesByEndpointLabelsChunkWriterTest : public TuringTest { +protected: + void initialize() override { + _jobSystem = std::make_unique(); + _jobSystem->init(); + + _graph = Graph::create(); + SimpleGraph::createSimpleGraph(_graph.get()); + } + + void terminate() override { + _jobSystem->terminate(); + } + + void labelSet(std::string_view name, LabelSet& labelset) const { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + + const std::optional label = reader.getMetadata().labels().get(name); + ASSERT_TRUE(label.has_value()); + + labelset.set(*label); + } + + // The rows of the single-end writer over the same index whose other end carries otherLabels. + template + void expectedRows(const LabelSet& indexedLabels, const LabelSet& otherLabels, std::vector& rows) const { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + + SingleEndChunkWriterType chunkWriter(reader.getView(), LabelSetHandle(indexedLabels)); + std::vector indexedRows; + scanInChunks(chunkWriter, ChunkConfig::CHUNK_SIZE, indexedRows); + + const LabelSetHandle otherHandle(otherLabels); + for (const ScannedRow& row : indexedRows) { + const LabelSetHandle otherEndLabels = reader.getNodeLabelSet(NodeID {std::get<3>(row)}); + if (otherEndLabels.isValid() && otherEndLabels.hasAtLeastLabels(otherHandle)) { + rows.push_back(row); + } + } + } + + template + void scannedRows(const LabelSet& indexedLabels, const LabelSet& otherLabels, size_t chunkSize, std::vector& rows) const { + const FrozenCommitTx transaction = _graph->openTransaction(); + const GraphReader reader = transaction.readGraph(); + + ChunkWriterType chunkWriter(reader.getView(), LabelSetHandle(indexedLabels), LabelSetHandle(otherLabels)); + scanInChunks(chunkWriter, chunkSize, rows); + } + + void deleteEdge(EdgeID edgeID) { + GraphWriter writer(_graph.get(), _jobSystem.get()); + writer.deleteEdge(edgeID); + writer.submit(); + } + + std::unique_ptr _jobSystem; + std::unique_ptr _graph; +}; + +// The 15 Person -> Interest edges sit among the Person out-edges with the KNOWS_WELL edges +// between Person nodes, so a fill of 1 to 4 rows stops mid-range, and some ranges end on +// edges the other end's labels drop. +TEST_F(ScanEdgesByEndpointLabelsChunkWriterTest, outScanEmitsEveryRowWhateverTheChunkSize) { + LabelSet person; + LabelSet interest; + labelSet("Person", person); + labelSet("Interest", interest); + + std::vector expected; + expectedRows(person, interest, expected); + ASSERT_EQ(expected.size(), 15u); + + for (size_t chunkSize = 1; chunkSize <= 4; chunkSize++) { + std::vector rows; + scannedRows(person, interest, chunkSize, rows); + + EXPECT_EQ(rows, expected) << "chunk size " << chunkSize; + } +} + +TEST_F(ScanEdgesByEndpointLabelsChunkWriterTest, inScanEmitsEveryRowWhateverTheChunkSize) { + LabelSet person; + LabelSet interest; + labelSet("Person", person); + labelSet("Interest", interest); + + std::vector expected; + expectedRows(interest, person, expected); + ASSERT_EQ(expected.size(), 15u); + + for (size_t chunkSize = 1; chunkSize <= 4; chunkSize++) { + std::vector rows; + scannedRows(interest, person, chunkSize, rows); + + EXPECT_EQ(rows, expected) << "chunk size " << chunkSize; + } +} + +TEST_F(ScanEdgesByEndpointLabelsChunkWriterTest, outScanDropsADeletedEdgeFromEveryColumn) { + LabelSet person; + LabelSet interest; + labelSet("Person", person); + labelSet("Interest", interest); + + std::vector beforeDelete; + expectedRows(person, interest, beforeDelete); + ASSERT_FALSE(beforeDelete.empty()); + + const ScannedRow deletedRow = beforeDelete.front(); + deleteEdge(EdgeID {std::get<1>(deletedRow)}); + + std::vector expected = beforeDelete; + expected.erase(expected.begin()); + + for (const size_t chunkSize : {size_t {1}, size_t {3}, ChunkConfig::CHUNK_SIZE}) { + std::vector rows; + scannedRows(person, interest, chunkSize, rows); + + EXPECT_EQ(rows, expected) << "chunk size " << chunkSize; + } +} + +TEST_F(ScanEdgesByEndpointLabelsChunkWriterTest, inScanDropsADeletedEdgeFromEveryColumn) { + LabelSet person; + LabelSet interest; + labelSet("Person", person); + labelSet("Interest", interest); + + std::vector beforeDelete; + expectedRows(interest, person, beforeDelete); + ASSERT_FALSE(beforeDelete.empty()); + + const ScannedRow deletedRow = beforeDelete.front(); + deleteEdge(EdgeID {std::get<1>(deletedRow)}); + + std::vector expected = beforeDelete; + expected.erase(expected.begin()); + + for (const size_t chunkSize : {size_t {1}, size_t {3}, ChunkConfig::CHUNK_SIZE}) { + std::vector rows; + scannedRows(interest, person, chunkSize, rows); + + EXPECT_EQ(rows, expected) << "chunk size " << chunkSize; + } +}