CASSANDRA-21530 Remove duplicate build scripts, point AGENTS.md to existing project build scripts

.build/build-jars.sh

@@ -17,6 +17,25 @@
 
 # temporary between CASSANDRA-18133 and CASSANDRA-18594
 
+print_help() {
+  echo "Usage: $0 [-c|--clean] [-s|--summary] [-h|--help]"
+  echo "   -c, --clean    Remove locally created artifacts (ant clean) before building"
+  echo "   -s, --summary  Print a summary of failures instead of the full ant output"
+  echo "   -h, --help     Print help"
+}
+
+# arguments, with defaults
+clean=false
+summary=false
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -c|--clean)   clean=true; shift ;;
+    -s|--summary) summary=true; shift ;;
+    -h|--help)    print_help; exit 0 ;;
+    *)            echo >&2 "Unknown argument $1"; print_help >&2; exit 1 ;;
+  esac
+done
+
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)"
 
@@ -25,6 +44,18 @@ command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1
 [ -d "${CASSANDRA_DIR}" ] || { echo >&2 "Directory ${CASSANDRA_DIR} must exist"; exit 1; }
 [ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; }
 
+# run ant, summarizing failures when --summary (summary exit code mirrors the build)
+run_ant() {
+  if ${summary}; then
+    ant -f "${CASSANDRA_DIR}/build.xml" "$@" 2>&1 | "${CASSANDRA_DIR}/.build/sh/ant-log-summary.py" -
+  else
+    ant -f "${CASSANDRA_DIR}/build.xml" "$@"
+  fi
+}
+
 # execute
-ant -f "${CASSANDRA_DIR}/build.xml" jar
+if ${clean}; then
+  run_ant clean
+fi
+run_ant jar
 exit $?

.build/check-code.sh

@@ -15,6 +15,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+print_help() {
+  echo "Usage: $0 [-s|--summary] [-h|--help]"
+  echo "   -s, --summary  Print a summary of failures instead of the full ant output"
+  echo "   -h, --help     Print help"
+}
+
+# arguments, with defaults
+summary=false
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -s|--summary) summary=true; shift ;;
+    -h|--help)    print_help; exit 0 ;;
+    *)            echo >&2 "Unknown argument $1"; print_help >&2; exit 1 ;;
+  esac
+done
+
 # variables, with defaults
 [ "x${CASSANDRA_DIR}" != "x" ] || { CASSANDRA_DIR="$(dirname -- "$0")/.."; }
 
@@ -24,5 +40,12 @@ command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1
 [ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; }
 
 # execute. memory needs to fit within the specified container size, see .jenkins/Jenkinsfile
-ANT_OPTS="-Xmx2g -XX:+PrintClassHistogram -XX:OnOutOfMemoryError='kill -QUIT %p'" ant -f "${CASSANDRA_DIR}/build.xml" check # dependency-check # FIXME dependency-check now requires NVD key downloaded first
+# dependency-check # FIXME dependency-check now requires NVD key downloaded first
+export ANT_OPTS="-Xmx2g -XX:+PrintClassHistogram -XX:OnOutOfMemoryError='kill -QUIT %p'"
+if ${summary}; then
+  # summarize failures; the summary's exit code mirrors the build
+  ant -f "${CASSANDRA_DIR}/build.xml" check 2>&1 | "${CASSANDRA_DIR}/.build/sh/ant-log-summary.py" -
+else
+  ant -f "${CASSANDRA_DIR}/build.xml" check
+fi
 exit $?

.build/run-tests.sh

@@ -31,21 +31,20 @@ set -o pipefail
 # target types
 TARGET_TYPES="build_dtest_jars stress-test fqltool-test sstableloader-test microbench microbench-test test-burn long-test cqlsh-test simulator-dtest test test-cdc test-compression test-oa test-system-keyspace-directory test-latest jvm-dtest jvm-dtest-upgrade jvm-dtest-novnode jvm-dtest-upgrade-novnode"
 
-# pre-conditions
+error() {
+  echo >&2 $2;
+  set -x
+  exit $1
+}
+
+# pre-conditions (error() must be defined above)
 command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
 command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
 command -v uuidgen >/dev/null 2>&1 || test -f /proc/sys/kernel/random/uuid || { error 1 "uuidgen needs to be installed"; }
 [ -d "${CASSANDRA_DIR}" ] || { error 1 "Directory ${CASSANDRA_DIR} must exist"; }
 [ -f "${CASSANDRA_DIR}/build.xml" ] || { error 1 "${CASSANDRA_DIR}/build.xml must exist"; }
 [ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; }
 
-
-error() {
-  echo >&2 $2;
-  set -x
-  exit $1
-}
-
 print_help() {
   echo "Usage: $0 [-a|-t|-c|-e|-i|-b|-s|-h]"
   echo "   -a Test target type: ${TARGET_TYPES}"
@@ -54,13 +53,12 @@ print_help() {
   echo "   -b Specify the base git branch for comparison when determining changed tests to"
   echo "      repeat. Defaults to ${BASE_BRANCH}. Note that this option is not used when"
   echo "      the '-a' option is specified."
-  echo "   -s Skip automatic detection of changed tests. Useful when you need to repeat a few ones,"
-  echo "      or when there are too many changed tests the CI env to handle."
   echo "   -e <key=value> Environment variables to be used in the repeated runs:"
   echo "                   -e REPEATED_TESTS_STOP_ON_FAILURE=false"
   echo "                   -e REPEATED_TESTS_COUNT=500"
   echo "                  If you want to specify multiple environment variables simply add multiple -e options."
   echo "   -i Ignore unknown environment variables"
+  echo "   -s Print a summary of failed tests instead of the full ant output"
   echo "   -h Print help"
 }
 
@@ -84,7 +82,8 @@ fi
 env_vars=""
 has_env_vars=false
 check_env_vars=true
-detect_changed_tests=true
+summary=false
+
 while getopts "a:t:c:e:ib:shj:" opt; do
   case $opt in
     a ) test_target="$OPTARG"
@@ -105,7 +104,7 @@ while getopts "a:t:c:e:ib:shj:" opt; do
         ;;
     i ) check_env_vars=false
         ;;
-    s ) detect_changed_tests=false
+    s ) summary=true
         ;;
     h ) print_help
         exit 0
@@ -331,7 +330,15 @@ _run_microbench() {
     local -r arch="$(uname -m)"
     local -r output_dir="${DIST_DIR}/test/output/${_target}/jdk${java_version}/${arch}/${_split_chunk//\//_}"
 
+    # microbench produces JMH json, not JUnit xml, so generate-test-report has
+    # nothing to summarise. Emit a "failed ..." line (matched by
+    # test-log-summary.py) so a benchmark failure is not reported as a pass in
+    # -s mode, where errexit is disabled and the ant failure would be swallowed.
+    set +o errexit
     ant $_target ${ANT_TEST_OPTS} -Dbuild.test.output.dir=${output_dir} -Dbenchmark.name="${benchmark_pattern}" -Dmaven.test.failure.ignore=true
+    local -r _ant_status=$?
+    set -o errexit
+    [[ ${_ant_status} -ne 0 ]] && echo "failed ${_target} ${_split_chunk}"
 
     # Post-process jmh-result.json to add jdk and arch parameters
     local jmh_result="${output_dir}/jmh-result.json"
@@ -357,7 +364,7 @@ _main() {
   local -r split_chunk="${chunk:-1/1}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
 
   # check split_chunk is compatible with target (if not a regexp)
-  if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
+  if [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
     case ${target} in
       "stress-test" | "fqltool-test" | "cqlsh-test" | "sstableloader-test")
           echo "Target ${target} does not support splits."
@@ -401,8 +408,12 @@ _main() {
   [ -d ${TMP_DIR} ] || mkdir -p "${TMP_DIR}"
   export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR} -Dbuild.test.output.dir=${DIST_DIR}/test/output/${target}"
 
-  # fresh virtualenv and test logs results everytime
-  [[ "/" == "${DIST_DIR}" ]] || rm -rf "${DIST_DIR}/test/{html,output,logs,reports}"
+  # fresh virtualenv and test logs results everytime.
+  # NB: braces are intentionally unquoted so brace-expansion produces the four
+  # paths; quoting them deletes a single literal '{html,output,logs,reports}'
+  # path (a no-op), leaving stale TEST-*.xml that generate-test-report would
+  # then merge into the summary.
+  [[ "/" == "${DIST_DIR}" ]] || rm -rf ${DIST_DIR}/test/{html,output,logs,reports}
 
   # cheap trick to ensure dependency libraries are in place. allows us to stash only project specific build artifacts.
   #  also recreate some of the non-build files we need
@@ -492,10 +503,30 @@ _main() {
       ;;
   esac
 
-  # merge all unit xml files into one, and print summary test numbers
-  ant -quiet -silent generate-test-report
+  # merge all unit xml files into one, and print summary test numbers.
+  # Pin the output/report dirs to DIST_DIR; otherwise the report scans ant's
+  # default ${basedir}/build/test/output and, when DIST_DIR is overridden,
+  # finds no results and reports a false pass.
+  ant -quiet -silent generate-test-report \
+    -Dbuild.test.output.dir="${DIST_DIR}/test/output" \
+    -Dbuild.test.report.dir="${DIST_DIR}/test/reports"
 
   popd  >/dev/null
 }
 
-_main "$@"
+if ${summary}; then
+  # Summarize the run: run-tests.sh keeps going past failing tests (ant still
+  # prints BUILD SUCCESSFUL), so the summary's exit code — derived from the
+  # test results — is what determines pass/fail here. stderr (where error()
+  # writes) is left untouched so setup failures stay visible.
+  # Disable errexit/pipefail so a non-zero _main doesn't pre-empt the summary.
+  set +o errexit +o pipefail
+  _main "$@" | "${CASSANDRA_DIR}/.build/sh/test-log-summary.py" -
+  main_status=${PIPESTATUS[0]}
+  summary_status=${PIPESTATUS[1]}
+  # a setup failure inside _main (error() exits non-zero) takes precedence
+  [[ ${main_status} -ne 0 ]] && exit ${main_status}
+  exit ${summary_status}
+else
+  _main "$@"
+fi