Merge pom_dlineage.xml into pom.xml, and make CI run the jar it builds

.github/scripts/run-all-demos.sh

@@ -43,8 +43,12 @@ trap 'rm -rf "$WORK"' EXIT
 read -r -a EXTRA_MVN <<< "${MVN_ARGS:-}"
 
 echo "== building classpath ${MVN_ARGS:+($MVN_ARGS)} =="
-# system-scope jars are on the compile classpath but not the runtime one, which
-# is why every demo here is documented as needing -Dexec.classpathScope=compile.
+# compile scope covers everything the demos need. It used to be load-bearing for
+# a different reason: three dependencies were <scope>system</scope> jars under
+# lib/, and system scope is on the compile classpath but not the runtime one,
+# which is why the demos were all documented as needing
+# -Dexec.classpathScope=compile. Those became ordinary Maven coordinates when
+# pom_dlineage.xml was merged into pom.xml, so that caveat no longer applies.
 if ! mvn -B -q ${EXTRA_MVN[@]+"${EXTRA_MVN[@]}"} dependency:build-classpath \
         -Dmdep.outputFile="$WORK/cp.txt" \
         -Dmdep.includeScope=compile > "$WORK/cp.log" 2>&1; then

.github/scripts/set-parser-version.sh

@@ -3,17 +3,21 @@
 # Read, set, or verify the GSP parser version everywhere this repository writes
 # it down.
 #
-# The version lives in five files. The root build and the dlineage build each
-# declare a ${gsp.core.version} property, and the three connector/ modules
-# hardcode the version in their gsqlparser dependency, because they are separate
-# builds with no parent to inherit a property from. Nothing made them agree, so
-# they could drift apart silently -- and a bump meant five hand edits, which is
-# most of why bumping felt expensive.
+# The version lives in four files. The root build declares it as a
+# ${gsp.core.version} property, and the three connector/ modules hardcode the
+# version in their gsqlparser dependency, because they are separate builds with
+# no parent to inherit a property from. Nothing made them agree, so they could
+# drift apart silently -- and a bump meant four hand edits, which is most of why
+# bumping felt expensive.
+#
+# It was five until pom_dlineage.xml was merged into pom.xml. That second POM
+# declared its own copy of the property for no reason other than that it was a
+# second POM.
 #
 # Usage:
 #   set-parser-version.sh              print the current version(s)
-#   set-parser-version.sh --check      exit 1 if the five disagree
-#   set-parser-version.sh 4.1.8        rewrite all five to 4.1.8
+#   set-parser-version.sh --check      exit 1 if the four disagree
+#   set-parser-version.sh 4.1.8        rewrite all four to 4.1.8
 #
 # Exit codes:
 #   0  success, or --check found them consistent
@@ -54,8 +58,6 @@ new = os.environ["NEW"]
 TARGETS = [
     ("pom.xml", "root build property",
      r"(?s)(<gsp\.core\.version>)([^<]+)(</gsp\.core\.version>)"),
-    ("pom_dlineage.xml", "dlineage build property",
-     r"(?s)(<gsp\.core\.version>)([^<]+)(</gsp\.core\.version>)"),
 ]
 for mod in ("oracleConnector", "snowflakeConnector", "sqlServerConnector"):
     TARGETS.append((

.github/scripts/smoke-dlineage-jar.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# Run the standalone data-lineage jar and check what it produced.
+#
+# This exists because building the dlineage demo proved nothing, twice. Until
+# 2026-07-28 it was a second POM, pom_dlineage.xml, and the only thing CI did
+# with it was `package`:
+#
+#   * issue #47 -- it was missing the JAXB dependency the root POM already
+#     carried. Compiles perfectly. Dies at runtime on every JDK after 8.
+#   * issue #46 -- the documented run command put external_lib/ on the
+#     classpath, a directory only the Windows .bat route ever creates, so
+#     nobody following the Maven instructions could run the thing at all.
+#
+# Both shipped through a green build. So this runs it.
+#
+# The assertions are about the OUTPUT, not the exit status, and that is the
+# whole point. In issue #47 the process created lineage.json, died before
+# writing a byte into it, and still exited 0. An existence check, or an exit
+# code check, would both have called that a pass. Non-empty and parseable is
+# the weakest check that would actually have caught it.
+#
+# Usage:
+#   smoke-dlineage-jar.sh [jar]
+#
+# Exit codes:
+#   0  the jar ran and produced valid, non-empty lineage in both formats
+#   1  anything else
+
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.." || exit 1
+
+JAR="${1:-target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar}"
+SQL="samples/dlineage/demo.sql"
+
+fail() { echo "::error::$*" >&2; exit 1; }
+
+[ -f "$JAR" ] || fail "$JAR was not produced by 'mvn package'"
+[ -f "$SQL" ] || fail "$SQL is missing; the smoke test has no input"
+
+OUT=$(mktemp -d)
+trap 'rm -rf "$OUT"' EXIT
+
+echo "running $JAR on $SQL"
+echo "java: $(java -version 2>&1 | head -1)"
+
+# --- JSON ------------------------------------------------------------------
+# No classpath, no version number, no external_lib. If this invocation ever
+# needs more than `java -jar`, the packaging has regressed and the README has
+# gone stale with it.
+java -jar "$JAR" /f "$SQL" /o "$OUT/lineage.json" /json \
+     /simpleShowRelationTypes fdd,fdr /filterRelationTypes fdd
+
+[ -s "$OUT/lineage.json" ] ||
+    fail "lineage.json is empty or missing: the tool ran but produced nothing (cf. issue #47)"
+
+python3 - "$OUT/lineage.json" <<'PY'
+import json, sys
+path = sys.argv[1]
+try:
+    d = json.load(open(path))
+except Exception as e:
+    sys.exit("lineage.json is not valid JSON: %s" % e)
+
+# The shape differs with /graph, so accept either place the relationships live.
+rel = d.get("relationships") or d.get("data", {}).get("sqlflow", {}).get("relationships", [])
+if not rel:
+    sys.exit("lineage.json parsed but carries no relationships; "
+             "the analyzer returned an empty result")
+print("ok: JSON output, %d relationships" % len(rel))
+PY
+
+# --- XML -------------------------------------------------------------------
+# A separate code path, through JAXB, and the one that actually broke. With
+# jaxb-runtime 2.3.3 on JDK 11+ the JSON above still succeeds while this writes
+# a 0-byte file and exits 0, so checking JSON alone is not enough. See the
+# jaxb-runtime comment in pom.xml for why the API and the runtime are
+# deliberately on different versions.
+java -jar "$JAR" /f "$SQL" /o "$OUT/lineage.xml"
+
+[ -s "$OUT/lineage.xml" ] ||
+    fail "lineage.xml is empty or missing: the JAXB output path failed (cf. issue #47)"
+
+python3 - "$OUT/lineage.xml" <<'PY'
+import sys, xml.dom.minidom
+path = sys.argv[1]
+try:
+    doc = xml.dom.minidom.parse(path)
+except Exception as e:
+    sys.exit("lineage.xml is not well-formed: %s" % e)
+
+rel = doc.getElementsByTagName("relationship")
+if not rel:
+    sys.exit("lineage.xml is well-formed but carries no <relationship> elements")
+print("ok: XML output, %d relationships" % len(rel))
+PY
+
+echo "ok: the standalone dlineage jar runs and produces lineage in both formats"

.github/workflows/build.yml

@@ -39,11 +39,11 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: maven
 
-      # The parser version is written in five files: the root and dlineage
-      # properties, and the three connector modules, which are separate builds
-      # with no parent to inherit from. Nothing used to make them agree, so a
-      # bump that missed one left a connector silently building against an older
-      # parser. Use .github/scripts/set-parser-version.sh <version> to move them.
+      # The parser version is written in four files: the root property, and the
+      # three connector modules, which are separate builds with no parent to
+      # inherit from. Nothing used to make them agree, so a bump that missed one
+      # left a connector silently building against an older parser. Use
+      # .github/scripts/set-parser-version.sh <version> to move them.
       - name: Parser version is consistent across all POMs
         run: .github/scripts/set-parser-version.sh --check
 
@@ -75,31 +75,24 @@ jobs:
           echo "$out"
           grep -q "syntax errors: 0" <<<"$out"
 
-      # pom_dlineage.xml is a second, separate build in this same directory, and
-      # nothing used to run it. It declared a <parent> of gudusoft:gsp_java,
-      # the private library reactor, so it only built for someone who had that
-      # POM installed locally; a plain clone got "Non-resolvable parent POM"
-      # before Maven read any of its own contents. Building it here is what
-      # stops that class of thing hiding again.
+      # The standalone lineage tool. It used to be a second POM,
+      # pom_dlineage.xml, and the step here only *built* it -- which is exactly
+      # how it shipped broken twice. Building proves nothing about a program:
+      # issue #47 was a missing JAXB dependency that compiles perfectly and dies
+      # at runtime on any JDK past 8, and issue #46 was a documented run command
+      # pointing at a directory the Maven route never creates. Both survived a
+      # green build. So this runs it and checks the output; see the script for
+      # why it asserts on the output rather than the exit status.
       #
-      # It also has to leave the root build alone: it declares a single source
-      # file, so sharing target/ with the root build makes the compiler plugin's
-      # incremental cleanup delete every class the root build just produced
-      # (issue #39). That is why it writes to target-dlineage/, and why the
-      # count check below is worth the two lines.
-      - name: Build the separate dlineage POM, and check it leaves target/ alone
-        shell: bash
-        run: |
-          set -euo pipefail
-          before=$(find target/classes -name '*.class' | wc -l)
-          mvn -B -f pom_dlineage.xml package -DskipTests
-          after=$(find target/classes -name '*.class' | wc -l)
-          echo "root classes before=$before after=$after"
-          if [ "$before" != "$after" ]; then
-            echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39"
-            exit 1
-          fi
-          ls -l target-dlineage/*.jar
+      # It runs on both matrix JDKs for the same reason: #47 does not reproduce
+      # on 8 at all, so a single-JDK check would have missed it.
+      #
+      # The issue-#39 guard that used to live here -- counting target/classes
+      # before and after, because the second POM's incremental cleanup would
+      # delete the root build's output -- is gone with the second POM. One POM
+      # cannot collide with itself.
+      - name: Run the standalone dlineage jar
+        run: .github/scripts/smoke-dlineage-jar.sh
 
   # The .bat scripts are the original Windows, no-Maven workflow: edit
   # setenv\setenv.bat, cd into a demo folder, run compile_<demo>.bat then

.github/workflows/nightly.yml

@@ -84,22 +84,12 @@ jobs:
         run: .github/scripts/check-test-results.sh
 
 
-      # A second, separate build in the same directory that nothing used to run.
-      # It carried a <parent> pointing at the private gudusoft:gsp_java reactor,
-      # so it only built where that POM happened to be installed. Also asserts it
-      # leaves the root build's target/classes alone (issue #39).
-      - name: Build the separate dlineage POM
-        shell: bash
-        run: |
-          set -euo pipefail
-          before=$(find target/classes -name '*.class' | wc -l)
-          mvn -B -f pom_dlineage.xml package -DskipTests
-          after=$(find target/classes -name '*.class' | wc -l)
-          if [ "$before" != "$after" ]; then
-            echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39"
-            exit 1
-          fi
-          ls -l target-dlineage/*.jar
+      # The standalone lineage tool, run rather than merely built. It was its own
+      # POM until 2026-07-28, and only ever compiled here, which is how it
+      # shipped a missing JAXB dependency (issue #47) and an unrunnable
+      # documented command (issue #46) through green builds.
+      - name: Run the standalone dlineage jar
+        run: .github/scripts/smoke-dlineage-jar.sh
 
       - name: Every demo starts
         run: .github/scripts/run-all-demos.sh --timeout 90
@@ -172,22 +162,13 @@ jobs:
         run: .github/scripts/check-test-results.sh
 
 
-      # A second, separate build in the same directory that nothing used to run.
-      # It carried a <parent> pointing at the private gudusoft:gsp_java reactor,
-      # so it only built where that POM happened to be installed. Also asserts it
-      # leaves the root build's target/classes alone (issue #39).
-      - name: Build the separate dlineage POM against the newest release
-        shell: bash
-        run: |
-          set -euo pipefail
-          before=$(find target/classes -name '*.class' | wc -l)
-          mvn -B -f pom_dlineage.xml package -DskipTests -Dgsp.core.version=${{ steps.ver.outputs.version }}
-          after=$(find target/classes -name '*.class' | wc -l)
-          if [ "$before" != "$after" ]; then
-            echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39"
-            exit 1
-          fi
-          ls -l target-dlineage/*.jar
+      # Same smoke run as the pinned job, against the jar "Compile everything"
+      # just built with -Dgsp.core.version=<newest release>. This is the check
+      # that would notice a new parser breaking data lineage at runtime while
+      # still compiling, which is precisely the failure mode a build-only step
+      # cannot see.
+      - name: Run the standalone dlineage jar against the newest release
+        run: .github/scripts/smoke-dlineage-jar.sh
 
       - name: Every demo starts
         env:
@@ -257,7 +238,7 @@ jobs:
             --title "Bump the parser from $pinned to $NEW" \
             --body "\`$NEW\` is the newest release on sqlparser.com. Tonight's nightly ran it through the same checks the pinned \`$pinned\` gets, and it passed all of them: the full test suite with no failures, every demo starting, and every case in \`.github/scripts/demo-cases.tsv\` producing its expected output.
 
-          This touches the five places the version is written -- \`pom.xml\`, \`pom_dlineage.xml\` and the three \`connector/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart.
+          This touches the four places the version is written -- \`pom.xml\` and the three \`connector/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart.
 
           Merge to move the demos to \`$NEW\`. Close to stay on \`$pinned\`; the nightly will go on testing \`$NEW\` and will not reopen this until a newer release appears.
 

.gitignore

@@ -57,7 +57,6 @@ rebel.xml
 
 # Maven target
 **/target
-target-dlineage
 
 *.iml
 
@@ -68,12 +67,9 @@ CLAUDE.md
 
 
 
-# generated by src/main/java/.../dlineage/buildJar.sh
-src/main/java/gudusoft/gsqlparser/demos/dlineage/class/
-src/main/java/gudusoft/gsqlparser/demos/dlineage/data_flow_analyzer.jar
-
-# populated by the .bat workflow, not vendored:
-#   mvn dependency:copy -Dartifact=com.gudusoft:gsqlparser:<ver> -DoutputDirectory=external_lib
+# populated by the .bat workflow, not vendored. setenv/fetch-parser.bat runs:
+#   mvn dependency:copy-dependencies -DoutputDirectory=external_lib -DincludeScope=runtime
+# Delete the directory to force a refetch after changing a dependency.
 external_lib/
 
 # javac output dir used by compile_<demo>.bat