[3.0] Adds a repeatable converter from template functions to Plates templates (part 1 of 2)

.dev/README.md

@@ -583,6 +583,54 @@ at.
 
 [baseline]: https://github.com/SimpleMachines/SMF/pull/9330
 
+## Converting templates to Plates
+
+The default theme's `*.template.php` files, each a set of `template_*()`
+functions that echo, convert mechanically into Plates templates, one file per
+sub-template in `Themes/default/templates/<Template>/`:
+
+```bash
+.dev/convert-templates.sh Who Recent
+.dev/convert-templates.sh --from <commit> --all --skip Stats
+```
+
+Every run starts again from the old files as the commit has them (`HEAD` by
+default), so it can be repeated as often as the converter changes, and the
+result depends on nothing else. `.dev/plates/convert.php` does the conversion;
+the script adds the handful of fix-ups the default theme needs, removes the
+empty lines the conversion leaves where HTML does not need them, and runs
+php-cs-fixer. The converter stops on anything it cannot convert faithfully,
+and warns about what a person should look at: a template that returns a
+value, a by-reference parameter, a call whose parameters it cannot find.
+
+Convert callers before the templates they call. A converted template reaches
+a sub-template of either kind through `$this->subTemplate()`, but an old one
+calling a `template_*()` function breaks once that function is gone.
+
+The point of converting mechanically is that the result can be checked
+mechanically. Capture every page the template snapshot suite renders, on both
+engines, before and after, and compare them:
+
+```bash
+.dev/capture-pages.sh before
+.dev/convert-templates.sh Who
+.dev/capture-pages.sh after
+.dev/compare-pages.sh before after -v
+.dev/compare-pages.sh before after --rendered -v
+```
+
+Straight out of the converter the pages match byte for byte, once sessions,
+tokens, times and counts are masked. After the tidying they match as a browser
+shows them: `--rendered` treats a run of whitespace as one space, except
+inside `<pre>`, `<textarea>` and `<script>`, which must still match exactly.
+The capture includes a few responses the snapshot suite does not cover:
+`ssi_examples.php`, XML responses, and `.dev/plates/probe.php`, which calls
+the template helpers the way a mod does.
+
+Capture both sides on the same forum, one straight after the other. The pages
+show counts and recent activity, so a forum that has been used in between
+reads differently for reasons that have nothing to do with the templates.
+
 ## Debugging SQL with the PostgreSQL log
 
 The `postgres` log is the best tool in the stack for tracking down a broken
@@ -671,6 +719,14 @@ compose.yaml                     the stack
 .dev/interrupt-upgrade.sh        kill an upgrade part way, report what recovery left
 .dev/compare-upgrade.sh          upgrade a 2.1 dump, install 3.0, diff the two
 .dev/schema-tool.php             read a database's shape, and compare readings
+
+.dev/convert-templates.sh        convert default theme templates to Plates, repeatably
+.dev/capture-pages.sh            save every snapshot page's HTML, on both engines
+.dev/compare-pages.sh            diff two captures, byte for byte or as rendered
+.dev/plates/convert.php          the converter: template_*() functions to Plates files
+.dev/plates/tidy.php             drop empty lines HTML does not need
+.dev/plates/compare.php          what compare-pages.sh runs
+.dev/plates/probe.php            call the template helpers the way a mod does
 ```
 
 `.dev/db.php` exists because the two runners reach the database differently.

.dev/capture-pages.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Saves the HTML of every page the template snapshot suite renders, on both
+# engines, plus a few responses the suite does not cover, so that two versions
+# of the templates can be compared page by page.
+#
+#   .dev/capture-pages.sh before                  into logs/pages-before/
+#   ... change the templates ...
+#   .dev/capture-pages.sh after
+#   .dev/compare-pages.sh before after -v
+#
+# The extra responses, fetched as a guest from $SMF_BOARDURL: ssi_examples.php,
+# the statistics centre's XML, the keep-alive and a help popup, and
+# .dev/plates/probe.php, which calls the template helpers the way a mod does.
+#
+# Capture both sides on the same forum, one straight after the other: the
+# pages show counts and recent activity, and a forum that has been used in
+# between reads differently for reasons that have nothing to do with the
+# templates.
+set -euo pipefail
+
+. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
+
+parse_runner_args "$@"
+
+LABEL=''
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+		--docker|--local) shift ;;
+		-h|--help) sed -n '2,19p' "${BASH_SOURCE[0]}"; exit 0 ;;
+		-*) die "unknown option: $1" ;;
+		*) LABEL="$1"; shift ;;
+	esac
+done
+
+[ -n "$LABEL" ] || die "name the capture, for example: $(basename -- "$0") before"
+
+cd "$BOARD_DIR"
+
+out="logs/pages-$LABEL"
+rm -rf "$out"
+
+SMF_SNAPSHOT_RAW="$out" "$DEV_DIR/test.sh" "--$SMF_RUNNER" --engine both --testsuite snapshot
+
+mkdir -p "$out/extras"
+cp .dev/plates/probe.php ./plates-probe.php
+trap 'rm -f ./plates-probe.php' EXIT
+
+fetch() {
+	curl -sS -o "$out/extras/$1.html" "$SMF_BOARDURL/$2"
+}
+
+fetch ssi_examples 'ssi_examples.php'
+fetch probe 'plates-probe.php'
+fetch stats_xml 'index.php?action=stats;xml;expand=202001'
+fetch stats_xml_now "index.php?action=stats;xml;expand=$(date +%Y%m)"
+fetch keepalive 'index.php?action=keepalive'
+fetch help_popup 'index.php?action=helpadmin;help=cal_enabled'
+
+log "captured $(find "$out" -name '*.html' | wc -l | tr -d ' ') pages in $out"

.dev/compare-pages.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Compares two captures made by .dev/capture-pages.sh and lists the pages that
+# differ. Exits non-zero if any do.
+#
+#   .dev/compare-pages.sh before after              byte for byte, after masking
+#   .dev/compare-pages.sh before after --rendered   as a browser shows them
+#   .dev/compare-pages.sh before after -v           and where each page differs
+#
+# What is masked, and what --rendered lets through, is described at the top of
+# .dev/plates/compare.php.
+set -euo pipefail
+
+. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
+
+parse_runner_args "$@"
+
+LABELS=()
+FLAGS=()
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+		--docker|--local) shift ;;
+		-h|--help) sed -n '2,11p' "${BASH_SOURCE[0]}"; exit 0 ;;
+		-*) FLAGS+=("$1"); shift ;;
+		*) LABELS+=("$1"); shift ;;
+	esac
+done
+
+[ ${#LABELS[@]} -eq 2 ] || die "name two captures, for example: $(basename -- "$0") before after"
+
+run_php .dev/plates/compare.php "logs/pages-${LABELS[0]}" "logs/pages-${LABELS[1]}" ${FLAGS[@]+"${FLAGS[@]}"}

.dev/convert-templates.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Converts the default theme's templates to Plates, from the *.template.php
+# files as a commit has them. Safe to run as often as needed: each run starts
+# again from the old file, so the result depends only on the commit and the
+# converter.
+#
+#   .dev/convert-templates.sh Who Recent                 from HEAD
+#   .dev/convert-templates.sh --from ea5ae213e Who       from another commit
+#   .dev/convert-templates.sh --from ea5ae213e --all --skip Stats
+#
+# For each template it restores the old file, converts it with
+# .dev/plates/convert.php into Themes/default/templates/<Template>/, applies
+# the few fix-ups the converter cannot make on its own, removes the old file,
+# drops the empty lines the conversion leaves where HTML does not need them
+# (.dev/plates/tidy.php), and runs php-cs-fixer over the result.
+#
+# Convert callers before the templates they call: a converted template reaches
+# either kind through $this->subTemplate(), while an old one calling a
+# template_*() function breaks once that function is gone.
+#
+# Needs git and perl on the host; PHP runs wherever $SMF_RUNNER says.
+set -euo pipefail
+
+. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
+
+parse_runner_args "$@"
+
+FROM='HEAD'
+ALL=''
+SKIP=()
+TEMPLATES=()
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+		--from) FROM="$2"; shift 2 ;;
+		--from=*) FROM="${1#*=}"; shift ;;
+		--all) ALL=1; shift ;;
+		--skip) SKIP+=("$2"); shift 2 ;;
+		--skip=*) SKIP+=("${1#*=}"); shift ;;
+		--docker|--local) shift ;;
+		-h|--help) sed -n '2,22p' "${BASH_SOURCE[0]}"; exit 0 ;;
+		-*) die "unknown option: $1" ;;
+		*) TEMPLATES+=("$1"); shift ;;
+	esac
+done
+
+cd "$BOARD_DIR"
+
+theme='Themes/default'
+
+# The old templates, so the converter knows the parameter names of templates
+# that have already been converted. Inside the checkout, where the container
+# can read them too; ignored by git.
+signatures='.dev/plates/signatures'
+rm -rf "$signatures"
+mkdir -p "$signatures"
+trap 'rm -rf "$signatures"' EXIT
+
+while IFS= read -r path; do
+	git show "$FROM:$path" > "$signatures/$(basename -- "$path")"
+done < <(git ls-tree --name-only "$FROM" "$theme/" | grep '\.template\.php$')
+
+if [ -n "$ALL" ]; then
+	for path in "$signatures"/*.template.php; do
+		TEMPLATES+=("$(basename -- "$path" .template.php)")
+	done
+fi
+
+files=()
+
+for t in ${TEMPLATES[@]+"${TEMPLATES[@]}"}; do
+	for s in ${SKIP[@]+"${SKIP[@]}"}; do
+		[ "$t" = "$s" ] && continue 2
+	done
+
+	[ -f "$signatures/$t.template.php" ] || die "$FROM has no $theme/$t.template.php"
+
+	cp "$signatures/$t.template.php" "$theme/$t.template.php"
+	rm -rf "$theme/templates/$t"
+	files+=("$theme/$t.template.php")
+done
+
+[ ${#files[@]} -gt 0 ] || die "nothing to convert; name the templates, or pass --all"
+
+# The fallbacks Xml defines for when no index template is loaded are defined
+# with the other legacy functions in Sources/TemplateFunctions.php.
+if [ -f "$theme/Xml.template.php" ]; then
+	perl -0pi -e 's{/\*\n \* This is just to hold off some errors if people are stupid\.\n \*/\nif \(!function_exists\(\x27template_button_strip\x27\)\) \{\n.*?\n\}\n\n}{}s' "$theme/Xml.template.php"
+
+	if grep -q 'hold off some errors' "$theme/Xml.template.php"; then
+		die "could not remove Xml's fallbacks"
+	fi
+fi
+
+run_php .dev/plates/convert.php --signatures="$signatures" "${files[@]}"
+
+# Applies a fix-up with perl, and fails if what it should have removed is still there.
+fixup() {
+	local file=$1 perl=$2 check=$3
+
+	[ -f "$file" ] || return 0
+	perl -0pi -e "$perl" "$file"
+
+	if grep -qF -- "$check" "$file"; then
+		die "fix-up did not apply to $file: $check"
+	fi
+}
+
+# The moderation centre's blocks, called through a variable inside an echo.
+fixup "$theme/templates/ModerationCenter/moderation_center.php" \
+	's/<\?php \$block_function = \x27template_\x27 \. \$block; \?>//; s/function_exists\(\$block_function\) \? \$block_function\(\) : \x27\x27/\$this->hasSubTemplate(\$block) ? \$this->subTemplate(\$block) : \x27\x27/' \
+	'$block_function'
+
+# Returned its HTML for a list column; a Plates template outputs it instead.
+fixup "$theme/templates/ModerationCenter/user_watch_post_callback.php" \
+	's/<\?php return \$output_html; \?>/<?php echo \$output_html; ?>/' \
+	'return $output_html'
+
+# The quick buttons are always output; template_quickbuttons() returns them when asked.
+fixup "$theme/templates/index/quickbuttons.php" \
+	's/<\?php if \(\$output_method == \x27echo\x27\): \?><\?= \$output \?><\?php else: \?><\?php return \$output; \?><\?php endif; \?>/<?= \$output ?>/; s/ \* \@param string \$output_method [^\n]*\n/ * \@param string \$output_method Ignored: the buttons are output, and template_quickbuttons() returns them when asked.\n/; s/ \* \@return void\|string [^\n]*\n//' \
+	'return $output'
+
+converted=()
+
+for file in "${files[@]}"; do
+	t=$(basename -- "$file" .template.php)
+
+	rm "$file"
+	git rm -q --cached --ignore-unmatch "$file"
+
+	while IFS= read -r path; do
+		converted+=("$path")
+	done < <(find "$theme/templates/$t" -name '*.php' ! -name index.php | sort)
+done
+
+run_php .dev/plates/tidy.php "${converted[@]}"
+run_cmd php vendor/bin/php-cs-fixer fix --allow-risky=yes --quiet --config .php-cs-fixer.dist.php "${converted[@]}"
+
+log "converted ${#files[@]} templates into ${#converted[@]} files"