diff --git a/.github/scripts/run-all-demos.sh b/.github/scripts/run-all-demos.sh index 3c0fda63..72405f9b 100755 --- a/.github/scripts/run-all-demos.sh +++ b/.github/scripts/run-all-demos.sh @@ -43,8 +43,12 @@ trap 'rm -rf "$WORK"' EXIT read -r -a EXTRA_MVN <<< "${MVN_ARGS:-}" echo "== building classpath ${MVN_ARGS:+($MVN_ARGS)} ==" -# system-scope jars are on the compile classpath but not the runtime one, which -# is why every demo here is documented as needing -Dexec.classpathScope=compile. +# compile scope covers everything the demos need. It used to be load-bearing for +# a different reason: three dependencies were system jars under +# lib/, and system scope is on the compile classpath but not the runtime one, +# which is why the demos were all documented as needing +# -Dexec.classpathScope=compile. Those became ordinary Maven coordinates when +# pom_dlineage.xml was merged into pom.xml, so that caveat no longer applies. if ! mvn -B -q ${EXTRA_MVN[@]+"${EXTRA_MVN[@]}"} dependency:build-classpath \ -Dmdep.outputFile="$WORK/cp.txt" \ -Dmdep.includeScope=compile > "$WORK/cp.log" 2>&1; then diff --git a/.github/scripts/set-parser-version.sh b/.github/scripts/set-parser-version.sh index f8ec0da9..8fcdf525 100755 --- a/.github/scripts/set-parser-version.sh +++ b/.github/scripts/set-parser-version.sh @@ -3,17 +3,21 @@ # Read, set, or verify the GSP parser version everywhere this repository writes # it down. # -# The version lives in five files. The root build and the dlineage build each -# declare a ${gsp.core.version} property, and the three connector/ modules -# hardcode the version in their gsqlparser dependency, because they are separate -# builds with no parent to inherit a property from. Nothing made them agree, so -# they could drift apart silently -- and a bump meant five hand edits, which is -# most of why bumping felt expensive. +# The version lives in four files. The root build declares it as a +# ${gsp.core.version} property, and the three connector/ modules hardcode the +# version in their gsqlparser dependency, because they are separate builds with +# no parent to inherit a property from. Nothing made them agree, so they could +# drift apart silently -- and a bump meant four hand edits, which is most of why +# bumping felt expensive. +# +# It was five until pom_dlineage.xml was merged into pom.xml. That second POM +# declared its own copy of the property for no reason other than that it was a +# second POM. # # Usage: # set-parser-version.sh print the current version(s) -# set-parser-version.sh --check exit 1 if the five disagree -# set-parser-version.sh 4.1.8 rewrite all five to 4.1.8 +# set-parser-version.sh --check exit 1 if the four disagree +# set-parser-version.sh 4.1.8 rewrite all four to 4.1.8 # # Exit codes: # 0 success, or --check found them consistent @@ -54,8 +58,6 @@ new = os.environ["NEW"] TARGETS = [ ("pom.xml", "root build property", r"(?s)()([^<]+)()"), - ("pom_dlineage.xml", "dlineage build property", - r"(?s)()([^<]+)()"), ] for mod in ("oracleConnector", "snowflakeConnector", "sqlServerConnector"): TARGETS.append(( diff --git a/.github/scripts/smoke-dlineage-jar.sh b/.github/scripts/smoke-dlineage-jar.sh new file mode 100755 index 00000000..d9e89963 --- /dev/null +++ b/.github/scripts/smoke-dlineage-jar.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# Run the standalone data-lineage jar and check what it produced. +# +# This exists because building the dlineage demo proved nothing, twice. Until +# 2026-07-28 it was a second POM, pom_dlineage.xml, and the only thing CI did +# with it was `package`: +# +# * issue #47 -- it was missing the JAXB dependency the root POM already +# carried. Compiles perfectly. Dies at runtime on every JDK after 8. +# * issue #46 -- the documented run command put external_lib/ on the +# classpath, a directory only the Windows .bat route ever creates, so +# nobody following the Maven instructions could run the thing at all. +# +# Both shipped through a green build. So this runs it. +# +# The assertions are about the OUTPUT, not the exit status, and that is the +# whole point. In issue #47 the process created lineage.json, died before +# writing a byte into it, and still exited 0. An existence check, or an exit +# code check, would both have called that a pass. Non-empty and parseable is +# the weakest check that would actually have caught it. +# +# Usage: +# smoke-dlineage-jar.sh [jar] +# +# Exit codes: +# 0 the jar ran and produced valid, non-empty lineage in both formats +# 1 anything else + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." || exit 1 + +JAR="${1:-target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar}" +SQL="samples/dlineage/demo.sql" + +fail() { echo "::error::$*" >&2; exit 1; } + +[ -f "$JAR" ] || fail "$JAR was not produced by 'mvn package'" +[ -f "$SQL" ] || fail "$SQL is missing; the smoke test has no input" + +OUT=$(mktemp -d) +trap 'rm -rf "$OUT"' EXIT + +echo "running $JAR on $SQL" +echo "java: $(java -version 2>&1 | head -1)" + +# --- JSON ------------------------------------------------------------------ +# No classpath, no version number, no external_lib. If this invocation ever +# needs more than `java -jar`, the packaging has regressed and the README has +# gone stale with it. +java -jar "$JAR" /f "$SQL" /o "$OUT/lineage.json" /json \ + /simpleShowRelationTypes fdd,fdr /filterRelationTypes fdd + +[ -s "$OUT/lineage.json" ] || + fail "lineage.json is empty or missing: the tool ran but produced nothing (cf. issue #47)" + +python3 - "$OUT/lineage.json" <<'PY' +import json, sys +path = sys.argv[1] +try: + d = json.load(open(path)) +except Exception as e: + sys.exit("lineage.json is not valid JSON: %s" % e) + +# The shape differs with /graph, so accept either place the relationships live. +rel = d.get("relationships") or d.get("data", {}).get("sqlflow", {}).get("relationships", []) +if not rel: + sys.exit("lineage.json parsed but carries no relationships; " + "the analyzer returned an empty result") +print("ok: JSON output, %d relationships" % len(rel)) +PY + +# --- XML ------------------------------------------------------------------- +# A separate code path, through JAXB, and the one that actually broke. With +# jaxb-runtime 2.3.3 on JDK 11+ the JSON above still succeeds while this writes +# a 0-byte file and exits 0, so checking JSON alone is not enough. See the +# jaxb-runtime comment in pom.xml for why the API and the runtime are +# deliberately on different versions. +java -jar "$JAR" /f "$SQL" /o "$OUT/lineage.xml" + +[ -s "$OUT/lineage.xml" ] || + fail "lineage.xml is empty or missing: the JAXB output path failed (cf. issue #47)" + +python3 - "$OUT/lineage.xml" <<'PY' +import sys, xml.dom.minidom +path = sys.argv[1] +try: + doc = xml.dom.minidom.parse(path) +except Exception as e: + sys.exit("lineage.xml is not well-formed: %s" % e) + +rel = doc.getElementsByTagName("relationship") +if not rel: + sys.exit("lineage.xml is well-formed but carries no elements") +print("ok: XML output, %d relationships" % len(rel)) +PY + +echo "ok: the standalone dlineage jar runs and produces lineage in both formats" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8206d5e4..10fcfab9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,11 +39,11 @@ jobs: java-version: ${{ matrix.java }} cache: maven - # The parser version is written in five files: the root and dlineage - # properties, and the three connector modules, which are separate builds - # with no parent to inherit from. Nothing used to make them agree, so a - # bump that missed one left a connector silently building against an older - # parser. Use .github/scripts/set-parser-version.sh to move them. + # The parser version is written in four files: the root property, and the + # three connector modules, which are separate builds with no parent to + # inherit from. Nothing used to make them agree, so a bump that missed one + # left a connector silently building against an older parser. Use + # .github/scripts/set-parser-version.sh to move them. - name: Parser version is consistent across all POMs run: .github/scripts/set-parser-version.sh --check @@ -75,31 +75,24 @@ jobs: echo "$out" grep -q "syntax errors: 0" <<<"$out" - # pom_dlineage.xml is a second, separate build in this same directory, and - # nothing used to run it. It declared a of gudusoft:gsp_java, - # the private library reactor, so it only built for someone who had that - # POM installed locally; a plain clone got "Non-resolvable parent POM" - # before Maven read any of its own contents. Building it here is what - # stops that class of thing hiding again. + # The standalone lineage tool. It used to be a second POM, + # pom_dlineage.xml, and the step here only *built* it -- which is exactly + # how it shipped broken twice. Building proves nothing about a program: + # issue #47 was a missing JAXB dependency that compiles perfectly and dies + # at runtime on any JDK past 8, and issue #46 was a documented run command + # pointing at a directory the Maven route never creates. Both survived a + # green build. So this runs it and checks the output; see the script for + # why it asserts on the output rather than the exit status. # - # It also has to leave the root build alone: it declares a single source - # file, so sharing target/ with the root build makes the compiler plugin's - # incremental cleanup delete every class the root build just produced - # (issue #39). That is why it writes to target-dlineage/, and why the - # count check below is worth the two lines. - - name: Build the separate dlineage POM, and check it leaves target/ alone - shell: bash - run: | - set -euo pipefail - before=$(find target/classes -name '*.class' | wc -l) - mvn -B -f pom_dlineage.xml package -DskipTests - after=$(find target/classes -name '*.class' | wc -l) - echo "root classes before=$before after=$after" - if [ "$before" != "$after" ]; then - echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39" - exit 1 - fi - ls -l target-dlineage/*.jar + # It runs on both matrix JDKs for the same reason: #47 does not reproduce + # on 8 at all, so a single-JDK check would have missed it. + # + # The issue-#39 guard that used to live here -- counting target/classes + # before and after, because the second POM's incremental cleanup would + # delete the root build's output -- is gone with the second POM. One POM + # cannot collide with itself. + - name: Run the standalone dlineage jar + run: .github/scripts/smoke-dlineage-jar.sh # The .bat scripts are the original Windows, no-Maven workflow: edit # setenv\setenv.bat, cd into a demo folder, run compile_.bat then diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1129c4d9..c9f7df7e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -84,22 +84,12 @@ jobs: run: .github/scripts/check-test-results.sh - # A second, separate build in the same directory that nothing used to run. - # It carried a pointing at the private gudusoft:gsp_java reactor, - # so it only built where that POM happened to be installed. Also asserts it - # leaves the root build's target/classes alone (issue #39). - - name: Build the separate dlineage POM - shell: bash - run: | - set -euo pipefail - before=$(find target/classes -name '*.class' | wc -l) - mvn -B -f pom_dlineage.xml package -DskipTests - after=$(find target/classes -name '*.class' | wc -l) - if [ "$before" != "$after" ]; then - echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39" - exit 1 - fi - ls -l target-dlineage/*.jar + # The standalone lineage tool, run rather than merely built. It was its own + # POM until 2026-07-28, and only ever compiled here, which is how it + # shipped a missing JAXB dependency (issue #47) and an unrunnable + # documented command (issue #46) through green builds. + - name: Run the standalone dlineage jar + run: .github/scripts/smoke-dlineage-jar.sh - name: Every demo starts run: .github/scripts/run-all-demos.sh --timeout 90 @@ -172,22 +162,13 @@ jobs: run: .github/scripts/check-test-results.sh - # A second, separate build in the same directory that nothing used to run. - # It carried a pointing at the private gudusoft:gsp_java reactor, - # so it only built where that POM happened to be installed. Also asserts it - # leaves the root build's target/classes alone (issue #39). - - name: Build the separate dlineage POM against the newest release - shell: bash - run: | - set -euo pipefail - before=$(find target/classes -name '*.class' | wc -l) - mvn -B -f pom_dlineage.xml package -DskipTests -Dgsp.core.version=${{ steps.ver.outputs.version }} - after=$(find target/classes -name '*.class' | wc -l) - if [ "$before" != "$after" ]; then - echo "::error::pom_dlineage.xml changed the root build's target/classes ($before -> $after); see issue #39" - exit 1 - fi - ls -l target-dlineage/*.jar + # Same smoke run as the pinned job, against the jar "Compile everything" + # just built with -Dgsp.core.version=. This is the check + # that would notice a new parser breaking data lineage at runtime while + # still compiling, which is precisely the failure mode a build-only step + # cannot see. + - name: Run the standalone dlineage jar against the newest release + run: .github/scripts/smoke-dlineage-jar.sh - name: Every demo starts env: @@ -257,7 +238,7 @@ jobs: --title "Bump the parser from $pinned to $NEW" \ --body "\`$NEW\` is the newest release on sqlparser.com. Tonight's nightly ran it through the same checks the pinned \`$pinned\` gets, and it passed all of them: the full test suite with no failures, every demo starting, and every case in \`.github/scripts/demo-cases.tsv\` producing its expected output. - This touches the five places the version is written -- \`pom.xml\`, \`pom_dlineage.xml\` and the three \`connector/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart. + This touches the four places the version is written -- \`pom.xml\` and the three \`connector/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart. Merge to move the demos to \`$NEW\`. Close to stay on \`$pinned\`; the nightly will go on testing \`$NEW\` and will not reopen this until a newer release appears. diff --git a/.gitignore b/.gitignore index 92d60f4d..3938cb18 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,6 @@ rebel.xml # Maven target **/target -target-dlineage *.iml @@ -68,12 +67,9 @@ CLAUDE.md -# generated by src/main/java/.../dlineage/buildJar.sh -src/main/java/gudusoft/gsqlparser/demos/dlineage/class/ -src/main/java/gudusoft/gsqlparser/demos/dlineage/data_flow_analyzer.jar - -# populated by the .bat workflow, not vendored: -# mvn dependency:copy -Dartifact=com.gudusoft:gsqlparser: -DoutputDirectory=external_lib +# populated by the .bat workflow, not vendored. setenv/fetch-parser.bat runs: +# mvn dependency:copy-dependencies -DoutputDirectory=external_lib -DincludeScope=runtime +# Delete the directory to force a refetch after changing a dependency. external_lib/ # javac output dir used by compile_.bat diff --git a/README.md b/README.md index c1eb1ca4..2b049e93 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ SELECT a.id, b.name FROM ta a JOIN tb b ON a.id = b.id WHERE a.x > 1; SQL mvn -q exec:java -Dexec.mainClass=gudusoft.gsqlparser.demos.checksyntax.checksyntax \ - -Dexec.args="/f q.sql /t oracle" -Dexec.classpathScope=compile + -Dexec.args="/f q.sql /t oracle" ``` ```text @@ -53,7 +53,7 @@ Reformat it: ```bash mvn -q exec:java -Dexec.mainClass=gudusoft.gsqlparser.demos.formatsql.formatsql \ - -Dexec.args="q.sql" -Dexec.classpathScope=compile + -Dexec.args="q.sql" ``` ```text @@ -69,15 +69,17 @@ Argument conventions differ between demos: `checksyntax` takes `/f ` and `/t `, while `formatsql` takes a bare filename. Run any demo with no arguments and it prints its own usage line. -> **Use `-Dexec.classpathScope=compile`, not `runtime`.** `pom.xml` declares -> `simple-xml`, `fastjson` and a few other jars under `lib/` with -> `system`, since they have no public Maven coordinate. Maven's -> `runtime` classpath scope excludes `system`-scoped dependencies by design, so -> any demo that touches one of them (e.g. `dlineageBasic`, which uses -> `org.simpleframework.xml`, or `antiSQLInjection`, which uses -> `org.boris.expr`) fails with `NoClassDefFoundError` under `runtime` -> even though the jar is right there in `lib/`. `compile` scope includes them -> and works for every demo. +> **`-Dexec.classpathScope=compile` is no longer needed.** Every command in +> this README used to carry it, and older instructions elsewhere still do. It +> was a workaround: `pom.xml` declared `simple-xml`, `fastjson` and `expr4j` +> with `system`, pointing at jars committed under `lib/`, and +> Maven's `runtime` scope excludes `system`-scoped dependencies by design — so +> any demo touching one of them died with `NoClassDefFoundError` under the +> default scope even though the jar was right there on disk. +> +> All three are ordinary Maven dependencies now (see +> [Dependencies and security advisories](#dependencies-and-security-advisories) below) and `lib/` is gone, so the +> default scope works. The flag is harmless if you keep typing it. > **Every demo is `gudusoft.gsqlparser.demos..`.** The directory > path under `src/main/java/` *is* the package, for all 190 files, so you can @@ -117,13 +119,16 @@ To move every demo to a different parser build, run one command: .github/scripts/set-parser-version.sh 4.1.9 ``` -The version is written in **five** files, not one: the `${gsp.core.version}` -property in `pom.xml` and again in `pom_dlineage.xml`, plus a hardcoded -`` in each of the three `connector/*/pom.xml`, which are separate -builds with no parent to inherit a property from. Editing them by hand is how -one gets left behind, silently building against an older parser, so the script -owns all five and CI runs `set-parser-version.sh --check` to fail the build if -they ever disagree. +The version is written in **four** files, not one: the `${gsp.core.version}` +property in `pom.xml`, plus a hardcoded `` in each of the three +`connector/*/pom.xml`, which are separate builds with no parent to inherit a +property from. Editing them by hand is how one gets left behind, silently +building against an older parser, so the script owns all four and CI runs +`set-parser-version.sh --check` to fail the build if they ever disagree. + +(It was five until `pom_dlineage.xml` was folded into `pom.xml`. That POM +declared a second copy of the property for no reason beyond being a second +POM.) Available versions: @@ -168,7 +173,7 @@ is not used here on purpose: Instead the nightly does the bumping. Its `latest` job resolves the newest release, runs the whole suite and every demo against it, and **only if all of -that passes** opens a PR moving the five files. So you never edit a version: you +that passes** opens a PR moving the four files. So you never edit a version: you merge a PR that is already proven green. Given that old versions are deleted, treat a red `pinned` job with a green `latest` job as urgent rather than informational: it means the pinned release is gone. @@ -200,7 +205,7 @@ mvn -Plocal -Dgsp.core.version=4.1.5.9 compile mvn -q -Plocal -Dgsp.core.version=4.1.5.9 exec:java \ -Dexec.mainClass=gudusoft.gsqlparser.demos.checksyntax.checksyntax \ - -Dexec.args="/f q.sql /t oracle" -Dexec.classpathScope=compile + -Dexec.args="/f q.sql /t oracle" ``` This replaces the loop that existed while the demos were a vendored module of @@ -311,22 +316,34 @@ That was **57% of every source file in this repository**, and it duplicated same 365 top-level classes and nothing else, so the library shipped twice and javac quietly compiled the sources while the jar sat unused. -The sources are gone; the jar now supplies `org.boris.expr`. Only one file ever +The sources are gone; the jar supplies `org.boris.expr`. Only one file ever imported it (`GEval.java`), the `antiSQLInjection` tests cover the path, and -they pass against the jar. Two consequences worth knowing: - -- **The `antiSQLInjection` demo now needs `-Dexec.classpathScope=compile`**, for - the `system`-scope reason described above. It used to work under `runtime` - only because the classes happened to be compiled into `target/classes`. -- **`mvn package` no longer puts `org/boris/expr/**` inside the project jar.** - Nothing here consumes that jar as a library, so this only matters if you start - doing so. +they pass against the jar. One consequence worth knowing: **`mvn package` no +longer puts `org/boris/expr/**` inside the project jar.** Nothing here consumes +that jar as a library, so this only matters if you start doing so. (The shaded +`-dlineage` jar does contain it, along with every other runtime dependency — +that is what an uber jar is.) The dependency's coordinates were also wrong: it was declared as `tk.pratanumandal:expr4j`, a different library entirely, which would have pointed SBOM and vulnerability tooling at the wrong project. It now names what -is actually on disk, with a checksum recorded in `pom.xml` since the jar carries -no version metadata of its own. +is actually on disk, with a checksum recorded alongside it, since the jar +carries no version metadata of its own. + +It is also the one dependency here with no coordinate in any public repository — +searched and confirmed — so it is served from `lib-repo/`, a file-based Maven +repository inside this checkout, laid out exactly like `~/.m2/repository`. See +[`lib-repo/readme.md`](lib-repo/readme.md). It was a `system` jar +under `lib/` until 2026-07-28, which is worse than it sounds: `system` scope is +deprecated, invisible to Dependabot, **and skipped by `maven-shade-plugin`, +`maven-assembly-plugin` and `dependency:copy-dependencies` alike**, so it could +never appear in a packaged artifact or a generated classpath. That last part is +most of why the dlineage demo needed a hand-assembled `java -cp` +([#46](https://github.com/sqlparser/gsp_demo_java/issues/46)). + +If Gudu ever uploads it to `https://www.sqlparser.com/maven/`, delete `lib-repo/` +and declare it normally — publishing to a repository you control is the better +answer, and this project already resolves from that one. ### dbConnect was removed @@ -414,11 +431,18 @@ samples/dlineage/ 1 samples/callgraph/ 1 ``` -**Build output deleted.** `demos/dlineage/class/` is created and `rm -rf`'d by +**Build output deleted.** `demos/dlineage/class/` was created and `rm -rf`'d by `buildJar.sh` on every run, and `data-lineage-result.xml` was a generated -lineage report nothing referenced. Both are now in `.gitignore`. The source -manifest `demos/dlineage/MANIFEST.MF` stays, since `buildJar.sh` copies it — -its `Main-Class` had been missed by the package rename and is now correct. +lineage report nothing referenced. Both are now in `.gitignore`. + +`buildJar.sh`, `buildJar.bat` and `demos/dlineage/MANIFEST.MF` went with them on +2026-07-28. All three existed to hand-assemble the same `data_flow_analyzer.jar` +that `maven-shade-plugin` now produces from the normal build, and all three +named files that do not exist: `buildJar.sh` copied +`../../../../../lib/gudusoft.gsqlparser.jar`, which was never in `lib/` under +that name, and the manifest's `Class-Path` listed +`lib/gudusoft.gsqlparser.jar` and `lib/sqlflow-exporter.jar`, neither of which +survived. Nothing in CI ran any of them, which is why nobody noticed. What deliberately stays next to its demo: each demo's `readme.md`, its `compile_*.bat` / `run_*.bat` (which `cd` relative to their own location, so @@ -453,10 +477,28 @@ Two things worth keeping in mind for next time: 1.x release, and it is where the 1.x deserialization advisories were fixed. Leave it, or move to `fastjson2`; do not "upgrade" it within 1.x. -What still sits in `lib/` is exactly three jars that genuinely have no public -coordinate and are declared system-scope in both `pom.xml` and -`pom_dlineage.xml`: `expr4j` (see above), `simple-xml`, and `fastjson`. None of -them are currently flagged. +**`lib/` is now gone entirely, and with it the last `system`-scope dependency.** +Three jars were left in it, believed to have no public coordinate. Two of them +did: + +| jar | outcome | +|---|---| +| `simple-xml-2.7.1.jar` | on Maven Central as `org.simpleframework:simple-xml:2.7.1`, and **byte-for-byte identical** to the committed copy (`sha256 7a43d2d5…f4e429f9`). Declared normally; jar deleted. | +| `fastjson-1.2.83.jar` | on Maven Central as `com.alibaba:fastjson:1.2.83`, likewise byte-identical (`sha256 641a4d65…5fe692631d`). Declared normally; jar deleted. | +| `expr4j.jar` | genuinely not on Central under any groupId. Moved to `lib-repo/` as a real artifact, `org.boris:expr:0.0.0-vendored`. | + +Since the two Central jars are the same bytes, that swap changed nothing at +runtime and gained three things: Dependabot can see and patch them, packaging +plugins stop skipping them, and their **transitive** dependencies now resolve — +`simple-xml` pulls in `stax` and `xpp3`, which under `system` scope were simply +absent, waiting to surface as a `NoClassDefFoundError` on some code path nobody +had exercised yet. + +The rule that follows: **add a dependency by coordinate, not by file.** Check +Maven Central first — two of the three "no public coordinate" jars here were on +it all along. If it truly has no coordinate, publishing it to Gudu's own Maven +repository beats committing it; `lib-repo/` is the fallback for when nobody can +upload one. `sqlflow-exporter.jar` and `sqlflow-library.jar` were dropped on 2026-07-28. They supplied `gudusoft.dbadapter.TSQLDataSource`, which two demos used @@ -485,6 +527,11 @@ Vendored driver jars in particular should not come back. They are invisible to Dependabot for the same reason the system-scope entries above were, so they age in place with no bot to flag them. +The one exception, `lib-repo/`, is deliberately not a place to drop jars: it is a +Maven repository with a strict layout, every artifact needs a hand-written +`.pom`, and its readme says to check Central and Gudu's own repository first. +The friction is the point. + ## What is excluded from the build Two demos used to be excluded because they read metadata straight out of a @@ -500,10 +547,16 @@ metadata offline instead, which also let the two vendored jars behind it leave inspection already worked off that JSON string, so nothing else changed. `samples/columninspect/` has a runnable metadata file and script -What `pom.xml` still excludes from the default build: +**No main source is excluded from the build any more.** +`gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java` was the last one, and +its exclusion was circular: it was excluded because `pom_dlineage.xml` compiled +it separately, and that POM existed only to produce a standalone jar. With +`maven-shade-plugin` producing that jar from the one build, the exclusion had +nothing left to justify it. See +[The standalone dlineage tool](#the-standalone-dlineage-tool). + +What `pom.xml` still excludes is two **test** sources: -- `gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java` — not broken; - `pom_dlineage.xml` builds it on its own into `target-dlineage/` (issue #39) - `gudusoft/gsqlparser/visitorsTest/XmlSchemaValidationTest.java` and its `TestRunner` — they read an XSD from `../gsp_java_core/`, outside this repository. They compile fine; only that path stops them running, so they @@ -527,11 +580,19 @@ run_checksyntax.bat /f ..\..\..\..\..\..\..\q.sql /t oracle That is the whole workflow now. You no longer edit `setenv.bat` first: it keeps whatever `JAVA_HOME` is already set, and it fetches the parser for you. -**No parser jar is committed to this repository**, so the first script you run -pulls one down. `setenv.bat` calls `setenv/fetch-parser.bat`, which reads -`gsp.core.version` out of `pom.xml` and does a `mvn dependency:copy` into -`external_lib/`. It is a no-op once the jar is there, and `external_lib/` is -gitignored. +**No jar of any kind is committed to this repository**, so the first script you +run pulls them down. `setenv.bat` calls `setenv/fetch-parser.bat`, which runs +`mvn dependency:copy-dependencies` into `external_lib/`. It is a no-op once they +are there, and `external_lib/` is gitignored; delete that directory to force a +refetch after changing a dependency. + +It used to copy only the parser, because everything else the demos needed lived +in the committed `lib/` directory that this script put on the `CLASSPATH`. With +`lib/` gone, `copy-dependencies` brings the whole set — parser, `fastjson`, +`simple-xml`, `org.boris.expr` out of `lib-repo/`, JAXB, SnakeYAML — in one go, +straight from `pom.xml`. No version is named anywhere in the `.bat` family, so +it cannot drift from what the Maven build resolves. `CLASSPATH` is now a single +directory, `external_lib\*`. That means the `.bat` route needs Maven **once**, to fetch, and never again. That is a deliberate trade. Vendoring a parser under `lib/` is what let these @@ -541,8 +602,10 @@ jar in `lib/` was 3.1.1.0 while the demos had moved on to APIs like parser has. One artifact, resolved from one place, is worth a one-time Maven call. -`external_lib/` also comes before `lib/` on the classpath, so the fetched parser -wins over anything dropped into `lib/` later. +A side effect worth noting: `run_DataFlowAnalyzer.bat` gets JAXB now. It never +did before, because `lib/` had no copy of it, so the `.bat` route hit +[#47](https://github.com/sqlparser/gsp_demo_java/issues/47) on any modern JDK +just as the Maven route did. ### Verification status @@ -552,7 +615,7 @@ top of this file covers it. | phase | covered | current | |-------|---------|---------| -| Bootstrap | assert no parser jar is committed, then `fetch-parser.bat` pulls one into `external_lib/` | passing | +| Bootstrap | assert no parser jar is committed, then `fetch-parser.bat` pulls the parser and every other dependency into `external_lib/` | passing | | Compile | all **39** `compile_.bat` | **39/39** | | Launch | all **50** `run_.bat`, no arguments | **50/50** | | Run for real | 4 demos with arguments, output checked against an expected string | passing | @@ -661,68 +724,99 @@ three classes from the `TGetTableColumn.java` beside it and so broke any wildcard compile of that folder, which is exactly how `compile_gettablecolumns.bat` compiles it. -## Building the dlineage demo on its own +## The standalone dlineage tool -`pom_dlineage.xml` builds just `DataFlowAnalyzer` into its own jar, for use as a -standalone lineage tool: +`mvn package` produces a second, self-contained jar alongside the ordinary one: + +``` +target/gsp_demo_java-1.0-SNAPSHOT.jar the demos, needs a classpath +target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar standalone, runs on its own +``` + +The second is an executable uber jar built by `maven-shade-plugin` with every +runtime dependency inside it, so running the data-lineage analyser takes no +classpath at all: ```bash -mvn -f pom_dlineage.xml package +mvn package -DskipTests -java -cp "target-dlineage/gsp_demo_java_dlineage-1.0-SNAPSHOT.jar:external_lib/*:lib/*" \ - gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer \ - /f demo.sql /o lineage.json /json /graph \ +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /f samples/dlineage/demo.sql /o lineage.json /json /graph \ /simpleShowRelationTypes fdd,fdr /filterRelationTypes fdd ``` Other invocations it supports: `/t mssql`, `/t postgresql`, `/showER`, -`/filterRelationTypes fdd`. - -`pom_dlineage.xml` builds into `target-dlineage/`, a directory of its own, -rather than the root build's `target/`. It shares this repository's -`${project.basedir}` with `pom.xml` and declares only one source file, so -without a separate output directory the compiler plugin's incremental-build -cleanup would delete every `.class` file the root build produced that isn't -part of this smaller source set — wiping out a working root build before this -one even reaches its own build. See -[#39](https://github.com/sqlparser/gsp_demo_java/issues/39). - -**This build used to fail**, and had done since before the demo trees were -merged. The cause was the pinned jar, not the code: `pom_dlineage.xml` declared -`lib/gsqlparser-3.1.1.0.jar` on `system` scope, while `DataFlowAnalyzer` had -moved on to `getOption().setTraceTablePosition(...)` and -`ProcessUtility.generateColumnLevelLineageCsvSimple(...)`, neither of which a -3.1.1.0 jar has. It now resolves the same parser the root build does, and -builds and runs. - -**It then failed a second way, for anyone but us.** It kept a `` of -`gudusoft:gsp_java:1.0-SNAPSHOT`, the private library reactor, which is not -published anywhere. On a machine that had never installed that POM, Maven -stopped before reading the file at all: +`/filterRelationTypes fdd`. Omit `/json` to get XML instead. -``` -Non-resolvable parent POM for gudusoft:gsp_demo_java_dlineage:1.0-SNAPSHOT: -Could not find artifact gudusoft:gsp_java:pom:1.0-SNAPSHOT -``` +It carries every demo class, not just this one, so it doubles as a +no-setup way to run any of them: -The root `pom.xml` was cut loose from that parent when this repository was made -buildable outside Gudu; this file was missed, and it looked fine to everyone who -had the library checked out locally. The parent contributed nothing it needed: -two compiler properties it already declares, an empty ``, and one -`maven-antrun-plugin` marked `false`. It is gone, and the -build now resolves `com.gudusoft:gsqlparser` from Gudu's public Maven repository -and nothing else — verified against a completely empty local repository, where -no `gudusoft/` group directory is created at all. +```bash +java -cp target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + gudusoft.gsqlparser.demos.checksyntax.checksyntax /f q.sql /t oracle +``` -Both workflows now build it, and assert it left the root build's -`target/classes` untouched. Nothing ran it before, which is why two separate -breakages could sit in it unnoticed. +### What this replaced, and why it kept breaking + +Until 2026-07-28 this was a second POM, `pom_dlineage.xml`, built with +`mvn -f pom_dlineage.xml package`. It broke four times, and **every one of them +was invisible to a green build**, because until the very end nothing in CI ran +the thing — and for most of that time nothing even built it. + +1. **It pinned a parser jar from 2019.** It declared + `lib/gsqlparser-3.1.1.0.jar` on `system` scope while `DataFlowAnalyzer` had + moved on to `getOption().setTraceTablePosition(...)` and + `ProcessUtility.generateColumnLevelLineageCsvSimple(...)`, neither of which + that jar has. +2. **It kept a `` published nowhere**, `gudusoft:gsp_java:1.0-SNAPSHOT`, + the private library reactor. Anyone without that POM in their local + repository got `Non-resolvable parent POM` before Maven read a line of the + file. It looked fine to everyone at Gudu, who all have the library checked + out. The root `pom.xml` had been cut loose from that parent years earlier; + this one was missed. +3. **[#46](https://github.com/sqlparser/gsp_demo_java/issues/46) — the + documented run command could not work.** It put `external_lib/*` on the + classpath, a directory only the Windows `.bat` route ever creates, so anyone + following the Maven instructions in order hit `NoClassDefFoundError` on the + first command in the section. The jar Maven had actually downloaded sat in + `~/.m2`, under a filename containing a version number that changes with every + release, so it could not be guessed either. +4. **[#47](https://github.com/sqlparser/gsp_demo_java/issues/47) — it was + missing the JAXB dependency `pom.xml` already had.** JAXB left the JDK after + Java 8; `pom.xml` had carried `jakarta.xml.bind-api` and `jaxb-runtime` for + ages, and `pom_dlineage.xml` had neither. Since `pom.xml` *excluded* + `DataFlowAnalyzer` from its own build, this second POM was the only thing + that ever compiled the class, so no other build path could catch it. + +Merging it into `pom.xml` removes the category rather than the four instances. +There is no second POM to drift, no second parser version to forget, no second +dependency list to fall behind — and no +[#39](https://github.com/sqlparser/gsp_demo_java/issues/39) either, the bug +where the second POM's incremental-compile cleanup deleted the root build's +`target/classes` because both POMs shared `${project.basedir}`. One POM cannot +collide with itself, so `target-dlineage/` is gone too. + +The exclusion that made all this necessary was circular: `DataFlowAnalyzer` was +excluded from the root build *because* a separate POM compiled it, and that POM +existed to produce a standalone jar. `maven-shade-plugin` produces the +standalone jar from the one build, so nothing is excluded from `pom.xml` any +more. + +**What actually keeps it working now is that CI runs it.** +`.github/scripts/smoke-dlineage-jar.sh` executes the jar on +`samples/dlineage/demo.sql` in both `build.yml` and `nightly.yml`, on JDK 8 and +21, and asserts on the *output*: non-empty, parses, and contains lineage +relationships. That last part is not pedantry — in #47 the process created +`lineage.json`, died before writing a byte into it, and still exited `0`. A file +that exists and an exit code of zero were both, at that moment, lies. It checks +XML as well as JSON, because those are separate code paths and the JAXB one is +the one that broke. ## master and dev branches `master` is updated when a new GSP version is released on . The dev branches move faster and may not -compile against the released jar or the one under `lib/`. +compile against the released parser that `pom.xml` pins. ## Tutorials diff --git a/lib/expr4j.jar b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar similarity index 100% rename from lib/expr4j.jar rename to lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar diff --git a/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.md5 b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.md5 new file mode 100644 index 00000000..9c914d62 --- /dev/null +++ b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.md5 @@ -0,0 +1 @@ +9039506e3dabe4f6ef51c1cdda31ab6a diff --git a/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.sha1 b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.sha1 new file mode 100644 index 00000000..f5fadac5 --- /dev/null +++ b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.jar.sha1 @@ -0,0 +1 @@ +88f16f2d00c5783fc60358af0b0adb4f60422868 diff --git a/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom new file mode 100644 index 00000000..fcea056a --- /dev/null +++ b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom @@ -0,0 +1,42 @@ + + + + 4.0.0 + + org.boris + expr + 0.0.0-vendored + jar + + Boris Expr (vendored) + + The expression evaluator behind gudusoft.gsqlparser.demos.antiSQLInjection + (GEval). The jar carries no version metadata at all - a bare MANIFEST.MF + with no pom.properties - so the version above is a placeholder, not an + upstream release number. Identify the file by content instead: + + sha256 6267d9cb7cabcb24cf02d5bff1d11fdfea8c60c9ba1840dcb45359fb0165ca1b + 367 classfiles, Java 5 bytecode (major version 49) + + Note this is NOT tk.pratanumandal:expr4j, which the root pom.xml declared it + as for a long time despite the filename being the only thing they share. + That coordinate names a completely different library, and pointing SBOM and + vulnerability tooling at it would have made them scan the wrong project. + + diff --git a/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.md5 b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.md5 new file mode 100644 index 00000000..f414c6b0 --- /dev/null +++ b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.md5 @@ -0,0 +1 @@ +f8b37f9ad7c1bc02f023737b401a62e2 diff --git a/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.sha1 b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.sha1 new file mode 100644 index 00000000..a44f1b3b --- /dev/null +++ b/lib-repo/org/boris/expr/0.0.0-vendored/expr-0.0.0-vendored.pom.sha1 @@ -0,0 +1 @@ +9ec779d0a2e9f75b3adef627b3dd7c42893c6568 diff --git a/lib-repo/readme.md b/lib-repo/readme.md new file mode 100644 index 00000000..e6e874be --- /dev/null +++ b/lib-repo/readme.md @@ -0,0 +1,81 @@ +# `lib-repo/` — a file-based Maven repository + +This directory is a Maven repository, laid out exactly like `~/.m2/repository` +or Maven Central: `///`. It is +declared in the root `pom.xml` as + +```xml + + in-project-vendored + ${project.baseUri}lib-repo + +``` + +It holds exactly one artifact, and it exists for exactly one reason: that +artifact has no coordinate in any public repository. + +| coordinate | file | why it is here | +|---|---|---| +| `org.boris:expr:0.0.0-vendored` | `expr4j.jar`, renamed to the coordinate | not on Maven Central under any groupId; searched and confirmed | + +## Why this and not `system` scope + +These jars used to be `system` dependencies pointing at +`lib/*.jar`. That is deprecated, and it actively broke things: + +- **`system`-scope artifacts are omitted from every packaging plugin.** + `maven-shade-plugin`, `maven-assembly-plugin` and + `dependency:copy-dependencies` all skip them. So no packaged jar and no + generated classpath could ever contain them, which is why running the + dlineage demo needed a hand-assembled `java -cp` with a version number in it + ([#46](https://github.com/sqlparser/gsp_demo_java/issues/46)). +- **They are invisible to Dependabot**, so they age with nothing to flag them. +- They contribute no transitive dependencies, so a library that needs one + silently gets a `NoClassDefFoundError` at runtime instead of a resolution + error at build time. + +Declared as a real repository, the artifact resolves, packages, and appears on +generated classpaths like anything else. Dependabot still cannot advise on it — +nothing can advise on a jar with no upstream identity — but that is now the only +remaining downside rather than one of four. + +## Why not the other two jars that used to live in `lib/` + +`simple-xml-2.7.1.jar` and `fastjson-1.2.83.jar` are **on Maven Central, and +byte-for-byte identical to the copies this repository was carrying**: + +``` +7a43d2d5e488e0dac57a6de0284df4413a4733365239649309c4b693f4e429f9 simple-xml-2.7.1.jar +641a4d65ab32fbfdccd9c718e3f83ebc4caabdb5e4fe5b3d51527c5fe692631d fastjson-1.2.83.jar +``` + +so they are declared as ordinary `org.simpleframework:simple-xml:2.7.1` and +`com.alibaba:fastjson:1.2.83` dependencies and their vendored copies are gone. +Nothing belongs in here that has a public coordinate — check before adding. + +## If you add something + +Don't, if you can avoid it: prefer a public coordinate, and failing that, +prefer publishing to Gudu's own Maven repository at +`https://www.sqlparser.com/maven/`, which this project already resolves from. +That is where a jar like this really belongs; an in-project repository is the +fallback for when nobody can upload one. + +If you must, the layout has to be exact or Maven will not see it: + +``` +lib-repo//// + -.jar + -.pom # hand-written; list its own dependencies + *.sha1 *.md5 # optional, but Maven warns without them +``` + +Regenerate the checksums after any change: + +```bash +cd lib-repo/org/boris/expr/0.0.0-vendored +for f in *.jar *.pom; do + sha1sum "$f" | cut -d' ' -f1 > "$f.sha1" + md5sum "$f" | cut -d' ' -f1 > "$f.md5" +done +``` diff --git a/lib/fastjson-1.2.83.jar b/lib/fastjson-1.2.83.jar deleted file mode 100644 index fd843ae9..00000000 Binary files a/lib/fastjson-1.2.83.jar and /dev/null differ diff --git a/lib/simple-xml-2.7.1.jar b/lib/simple-xml-2.7.1.jar deleted file mode 100644 index ede93e64..00000000 Binary files a/lib/simple-xml-2.7.1.jar and /dev/null differ diff --git a/pom.xml b/pom.xml index f7950715..2655da5f 100644 --- a/pom.xml +++ b/pom.xml @@ -30,8 +30,28 @@ 4.1.9 - + + + in-project-vendored + ${project.baseUri}lib-repo + true + false + + + gudu-public-releases https://www.sqlparser.com/maven/ @@ -91,21 +111,33 @@ + org.simpleframework simple-xml 2.7.1 - system - ${project.basedir}/lib/simple-xml-2.7.1.jar - + com.alibaba fastjson 1.2.83 - system - ${project.basedir}/lib/fastjson-1.2.83.jar + removed; this artifact is now what actually supplies org.boris.expr. --> org.boris expr 0.0.0-vendored - system - ${project.basedir}/lib/expr4j.jar + and the JAXB API left the JDK after Java 8. + + The API stays on the javax.xml.bind namespace, not jakarta.xml.bind: + the parser calls javax.xml.bind.JAXBContext directly, so 2.3.3 (the + last javax-namespace release of the API) is a compatibility + requirement, not a version that has been left to rot. + + jaxb-runtime, though, MUST be 2.3.9 rather than the matching 2.3.3. + 2.3.3's optimised reflection path, + com.sun.xml.bind.v2.runtime.reflect.opt.Injector, calls + sun.misc.Unsafe.defineClass, which was removed in JDK 11. On any + newer JDK it logs + + SEVERE java.security.PrivilegedActionException: + NoSuchMethodException: sun.misc.Unsafe.defineClass + + and then XML2Model.saveXML fails with a NullPointerException and + "Output dataflow to xml failed.", so DataFlowAnalyzer's XML + output silently produced nothing on every JDK this project claims + to support. Verified on OpenJDK 21: 2.3.3 fails, 2.3.9 writes the + XML cleanly. Do not "align" these two version numbers. --> org.yaml snakeyaml @@ -176,7 +226,7 @@ org.glassfish.jaxb jaxb-runtime - 2.3.3 + 2.3.9 @@ -216,9 +266,22 @@ wildcard compile of that folder, which is how compile_gettablecolumns.bat compiles it. Deleted. --> - - gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.3 + + + dlineage-standalone + package + shade + + true + dlineage + + false + + + gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer + + + + + + + *:* + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + module-info.class + + + + + + + diff --git a/pom_dlineage.xml b/pom_dlineage.xml deleted file mode 100644 index e1bede16..00000000 --- a/pom_dlineage.xml +++ /dev/null @@ -1,166 +0,0 @@ - - - 4.0.0 - - - - gudusoft - gsp_demo_java_dlineage - Gudu Software Parser for Java Demo - dlineage only - Compile and run only dlineage demo - https://www.sqlparser.com - 1.0-SNAPSHOT - - - UTF-8 - UTF-8 - 1.8 - 1.8 - true - 4.1.9 - - - - - - gudu-public-releases - https://www.sqlparser.com/maven/ - - - - - - junit - junit - 4.13.2 - test - - - - com.gudusoft - gsqlparser - ${gsp.core.version} - - - - org.simpleframework - simple-xml - 2.7.1 - system - ${project.basedir}/lib/simple-xml-2.7.1.jar - - - - com.alibaba - fastjson - 1.2.83 - system - ${project.basedir}/lib/fastjson-1.2.83.jar - - - - com.github.junrar - junrar - 7.5.10 - - - - org.boris - expr - 0.0.0-vendored - system - ${project.basedir}/lib/expr4j.jar - - - - org.jdom - jdom2 - 2.0.6.1 - - - - org.yaml - snakeyaml - 2.2 - compile - - - - - - - ${project.basedir}/target-dlineage - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - ${maven.compiler.source} - ${maven.compiler.target} - - gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M5 - - true - - - - org.apache.maven.plugins - maven-jar-plugin - 3.3.0 - - - - gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer - - - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer - - - - - - - - diff --git a/setenv/fetch-parser.bat b/setenv/fetch-parser.bat index b655e559..40e61cd0 100644 --- a/setenv/fetch-parser.bat +++ b/setenv/fetch-parser.bat @@ -1,14 +1,27 @@ @echo off -REM # Fetches the General SQL Parser jar into external_lib\. +REM # Fetches the General SQL Parser jar, and every other dependency the demos +REM # need, into external_lib\. REM # -REM # No parser jar is committed to this repository. A fresh clone has none, by -REM # design: the parser is a released artifact resolved from Gudu's public Maven -REM # repository, and vendoring a copy is how this repository previously ended up -REM # compiling demos against a build years older than their source. +REM # No jar of any kind is committed to this repository. A fresh clone has +REM # none, by design: the parser is a released artifact resolved from Gudu's +REM # public Maven repository, and vendoring a copy is how this repository +REM # previously ended up compiling demos against a build years older than its +REM # own source. REM # -REM # Called automatically by setenv\setenv.bat when external_lib\ has no parser, -REM # so the compile_.bat / run_.bat scripts work on a fresh clone. -REM # Safe to run directly, and a no-op once the jar is there. +REM # This used to copy the parser alone, because everything else the demos +REM # needed sat in a committed lib\ directory that setenv.bat put on the +REM # CLASSPATH. Those jars were system dependencies, which is +REM # deprecated and which no packaging or classpath plugin can see, so they +REM # were replaced by ordinary Maven coordinates and lib\ was deleted. Maven +REM # is now the single source for all of them, and copy-dependencies brings +REM # the whole set down in one go: the parser, fastjson, simple-xml, +REM # org.boris.expr (out of lib-repo\, an in-project Maven repository), JAXB +REM # and SnakeYAML. +REM # +REM # Called automatically by setenv\setenv.bat when external_lib\ has no +REM # parser, so the compile_.bat / run_.bat scripts work on a +REM # fresh clone. Safe to run directly, and a no-op once the jars are there; +REM # delete external_lib\ to force a refetch after changing a dependency. REM # REM # Requires Maven on PATH, once. After that the .bat workflow needs nothing. @@ -17,35 +30,45 @@ setlocal REM # Run from the repository root regardless of where this was invoked from. cd /d "%~dp0.." +REM # The parser is the sentinel for "already fetched": it is the one artifact +REM # guaranteed to be in the set, and the one whose absence breaks everything. for %%f in (external_lib\gsqlparser-*.jar) do ( - echo Parser already present: %%f + echo Dependencies already present: %%f endlocal exit /b 0 ) -REM # Single source of truth for the version: gsp.core.version in pom.xml. -for /f "delims=" %%v in ('mvn -q help:evaluate -Dexpression^=gsp.core.version -DforceStdout 2^>nul') do set GSPVER=%%v +echo Fetching the parser and the other demo dependencies into external_lib\ ... +echo This needs Maven on your PATH, and runs once. -if "%GSPVER%"=="" ( +REM # No version is named here on purpose. copy-dependencies reads pom.xml, +REM # which is the single place gsp.core.version is written, so this cannot +REM # drift from what the Maven build resolves. includeScope=runtime keeps the +REM # test-only dependencies out; no .bat script runs tests. +call mvn -q dependency:copy-dependencies -DoutputDirectory=external_lib -DincludeScope=runtime +if errorlevel 1 ( echo. echo *************************** - echo Could not read gsp.core.version from pom.xml. - echo Is Maven on your PATH? The parser has to be fetched once: - echo mvn dependency:copy -Dartifact=com.gudusoft:gsqlparser:VERSION -DoutputDirectory=external_lib + echo Failed to fetch the demo dependencies. See the message above. + echo Is Maven on your PATH? The equivalent by hand is: + echo mvn dependency:copy-dependencies -DoutputDirectory=external_lib -DincludeScope=runtime echo *************************** echo. endlocal exit /b 1 ) -echo Fetching parser %GSPVER% into external_lib\ ... -call mvn -q dependency:copy -Dartifact=com.gudusoft:gsqlparser:%GSPVER% -DoutputDirectory=external_lib -if errorlevel 1 ( - echo Failed to fetch the parser. See the message above. +for %%f in (external_lib\gsqlparser-*.jar) do ( + echo Done. endlocal - exit /b 1 + exit /b 0 ) -echo Done. +echo. +echo *************************** +echo Maven reported success but no parser jar reached external_lib\. +echo Check the gsqlparser dependency in pom.xml. +echo *************************** +echo. endlocal -exit /b 0 +exit /b 1 diff --git a/setenv/setenv.bat b/setenv/setenv.bat index 33e4ab15..3f063a5b 100644 --- a/setenv/setenv.bat +++ b/setenv/setenv.bat @@ -6,7 +6,7 @@ REM # have to change the envrironment settings in only one location. REM # SET PATH FOR Native Libraries -set PATH=%PATH%;lib\;external_lib\ +set PATH=%PATH%;external_lib\ REM # set the Java home directory. REM # If JAVA_HOME is already set -- by CI, or by a developer who configured it @@ -24,17 +24,22 @@ set JAVAC_CMD="%JAVA_HOME%\bin\javac.exe" REM #Set the home directory of the GSP library set gspDemoHome=. -REM # No parser jar is committed to this repository, so fetch it into -REM # external_lib\ the first time. This is a no-op once it is there. +REM # No jars are committed to this repository, so fetch them into external_lib\ +REM # the first time. This is a no-op once they are there. REM # REM # A vendored parser is what previously let the demos compile against a build -REM # years older than their own source, so the jar is resolved from Gudu's -REM # public Maven repository instead and never checked in. +REM # years older than their own source, so every jar is resolved from Maven +REM # instead and none are checked in. for %%f in (%gspDemoHome%\external_lib\gsqlparser-*.jar) do goto :gspFound call "%gspDemoHome%\setenv\fetch-parser.bat" :gspFound -REM # set classpath to the GSP library Jar files and the database JDBC drivers. -REM # external_lib comes before lib so the fetched parser always wins over -REM # anything that may be dropped into lib\ later. -set CLASSPATH=.;%gspDemoHome%\build;%gspDemoHome%\external_lib\*;%gspDemoHome%\lib\* +REM # set classpath to the GSP library and the rest of the demo dependencies. +REM # +REM # There used to be a second entry here, %gspDemoHome%\lib\*, for a committed +REM # directory of system jars. Those became ordinary Maven +REM # dependencies when pom_dlineage.xml was merged into pom.xml, lib\ was +REM # deleted, and fetch-parser.bat now brings the same jars into external_lib\ +REM # via dependency:copy-dependencies. One directory, one source, nothing to +REM # keep in sync by hand. +set CLASSPATH=.;%gspDemoHome%\build;%gspDemoHome%\external_lib\* diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/MANIFEST.MF b/src/main/java/gudusoft/gsqlparser/demos/dlineage/MANIFEST.MF deleted file mode 100644 index fea1fd28..00000000 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/MANIFEST.MF +++ /dev/null @@ -1,4 +0,0 @@ -Manifest-Version: 1.0 -Class-Path: . lib/gudusoft.gsqlparser.jar lib/sqlflow-exporter.jar -Main-Class: gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer - diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.bat b/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.bat deleted file mode 100644 index 7ade0137..00000000 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.bat +++ /dev/null @@ -1,42 +0,0 @@ -@ECHO OFF -SETLOCAL enableDelayedExpansion - -SET cur_dir=%CD% -echo %cur_dir% - -SET qddemo=%cur_dir% - -IF EXIST %qddemo%\lib RMDIR %qddemo%\lib -IF NOT EXIST %qddemo%\lib MKDIR %qddemo%\lib -MKDIR %qddemo%\lib -XCOPY ..\..\..\..\..\lib\* %qddemo%\lib -XCOPY ..\..\..\..\..\external_lib\sqlflow-exporter.jar %qddemo%\lib - -SET qddemo_bin=%qddemo%\lib -SET qddemo_class=%qddemo%\class - -echo %qddemo_class% -echo %qddemo_bin% - -IF EXIST %qddemo_class% RMDIR %qddemo_class% -IF NOT EXIST %qddemo_class% MKDIR %qddemo_class% - -cd %cur_dir% -FOR /R %%b IN ( . ) DO ( -IF EXIST %%b/*.java SET JFILES=!JFILES! %%b/*.java -) - -MKDIR %qddemo_class%\lib -XCOPY %qddemo_bin% %qddemo_class%\lib -XCOPY %qddemo%\MANIFEST.MF %qddemo_class% - -cd %cur_dir% - - javac -d %qddemo_class% -encoding utf-8 -cp .;%qddemo_bin%\*; %JFILES% - -cd %qddemo_class% - jar -cvfm %qddemo%\data_flow_analyzer.jar %qddemo%\MANIFEST.MF * - -echo "successfully" - -pause diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.sh b/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.sh deleted file mode 100644 index 7bc1e568..00000000 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/buildJar.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -cur_dir=$(pwd) - -src_dir=$cur_dir -bin_dir=../../../../../lib -class_dir=$cur_dir/class - - -rm -rf $src_dir/sources.list -find $src_dir -name "*.java" > $src_dir/sources.list -cat $src_dir/sources.list - -mkdir $cur_dir/lib -cp $bin_dir/gudusoft.gsqlparser.jar $cur_dir/lib -rm -rf $class_dir -rm -rf $class_dir/lib -mkdir $class_dir -mkdir $class_dir/lib -cp $cur_dir/MANIFEST.MF $class_dir -cp -r $bin_dir/gudusoft.gsqlparser.jar $class_dir/lib - - - - -javac -d $class_dir -cp .:$bin_dir/gudusoft.gsqlparser.jar -g -sourcepath $src_dir @$src_dir/sources.list - -cd $class_dir -jar -cvfm $cur_dir/data_flow_analyzer.jar MANIFEST.MF * -rm -rf $class_dir