test.yaml

name: Test

on:
    push:
        branches: [ master, renovate/** ]
    pull_request:

jobs:
    build:
        name: Docs build
        runs-on: ubuntu-latest
        steps:
            -   name: Checkout Source code
                uses: actions/checkout@v6

            -   name: Use Node.js 24
                uses: actions/setup-node@v6
                with:
                    node-version: 24

            -   uses: apify/actions/pnpm-install@v1.1.2

            -   run: pnpm build
                env:
                    INTERCOM_APP_ID: ${{ secrets.INTERCOM_APP_ID }}
                    SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}

            -   name: Check llms.txt size
                run: npm run test:llms-size

            -   name: Install Nginx
                run: |
                    sudo apt-get update
                    sudo apt-get install -y nginx

            -   name: Start Docusaurus server
                run: |
                    nohup pnpm exec docusaurus serve --port 3000 --no-open &
                    sleep 5
                    curl -f http://localhost:3000 > /dev/null

            -   name: Start Nginx with project config
                run: |
                    PWD_PATH="$(pwd)"
                    cp nginx.conf nginx-test.conf
                    sed -i 's|https://apify.github.io/apify-docs|http://127.0.0.1:3000|g' nginx-test.conf
                    cat > default.conf <<EOF
                    worker_processes auto;
                    error_log ${PWD_PATH}/logs/error.log;
                    pid ${PWD_PATH}/logs/nginx.pid;
                    events {}
                    http {
                        access_log ${PWD_PATH}/logs/access.log;
                        include ${PWD_PATH}/nginx-test.conf;
                    }
                    EOF
                    mkdir -p "${PWD_PATH}/logs"
                    nginx -c "${PWD_PATH}/default.conf"
                    sleep 1

            -   name: Run header assertions
                run: |
                    set -euo pipefail
                    function assert_header() {
                        url=$1
                        header=$2
                        expected=$3
                        shift 3
                        extra_args=("$@")
                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
                        echo "→ $url → $actual"
                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
                    }

                    function assert_status() {
                        url=$1
                        expected=$2
                        shift 2
                        extra_args=("$@")
                        actual=$(curl -s -o /dev/null -w "%{http_code}" "${extra_args[@]}" "$url")
                        echo "→ $url → HTTP $actual"
                        [ "$actual" = "$expected" ] || (echo "❌ Expected HTTP $expected but got $actual for $url" && exit 1)
                    }

                    function assert_no_redirect() {
                        url=$1
                        shift
                        extra_args=("$@")
                        response=$(curl -s -D - -o /dev/null -w "\n%{http_code}" "${extra_args[@]}" "$url" 2>/dev/null)
                        status=$(echo "$response" | tail -1)
                        location=$(echo "$response" | grep -i "^location:" | tr -d '\r' || true)
                        echo "→ $url → HTTP $status ${location:+(${location})}"
                        if [ "$status" = "301" ] || [ "$status" = "302" ]; then
                            echo "❌ Got redirect for $url: $location" && exit 1
                        fi
                    }

                    # Like assert_header but follows redirects and asserts on the
                    # final response's Content-Type. Use for URLs that legitimately
                    # redirect before serving the content (e.g. child repo homepages).
                    # Only Content-Type is supported because curl's writeout variable
                    # %{content_type} is the only header it exposes after -L.
                    function assert_final_content_type() {
                        url=$1
                        expected=$2
                        shift 2
                        extra_args=("$@")
                        actual=$(curl -s -L -o /dev/null -w "%{content_type}" "${extra_args[@]}" "$url")
                        echo "→ $url → final Content-Type: $actual"
                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in final Content-Type for $url, got '$actual'" && exit 1)
                    }

                    # Asserts that the HTML at $url contains a
                    # <link rel="alternate" type="text/markdown" href="$expected_href">
                    # tag (attribute order/quoting may vary; we only require
                    # type=text/markdown and the expected href on the same
                    # <link> element). AI crawlers rely on this tag to find
                    # the .md counterpart of an HTML page.
                    function assert_html_md_alternate() {
                        url=$1
                        expected_href=$2
                        matched=$(curl -s "$url" | grep -oE '<link [^>]*type="?text/markdown[^>]*>' | grep -F "$expected_href" || true)
                        echo "→ $url → ${matched:-no match}"
                        if [ -z "$matched" ]; then
                            echo "❌ Expected <link rel=\"alternate\" type=\"text/markdown\" href=\"$expected_href\"> in HTML for $url"
                            exit 1
                        fi
                    }

                    echo "🧪 Checking open redirect protection..."
                    # Backslash URLs must not produce redirects (the redirect Location
                    # would contain \, which browsers normalize to /, creating
                    # protocol-relative URLs like //evil.com that redirect externally).
                    assert_no_redirect "http://localhost:8080///%5Cevil.com/"
                    assert_no_redirect "http://localhost:8080/%5Cevil.com/"
                    assert_no_redirect "http://localhost:8080///%5cevil.com/"
                    # Literal backslash (simulates CDN pre-decoding %5C before forwarding)
                    assert_no_redirect "http://localhost:8080" --request-target '/\evil.com/'
                    assert_no_redirect "http://localhost:8080" --request-target '///\evil.com/'
                    # Normal trailing-slash redirect must still work
                    assert_status "http://localhost:8080/platform/proxy/usage/" "302"

                    echo "🧪 Checking Nginx responses... (apify-docs)"
                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/platform/proxy/usage.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/img/docs-og.png" "Content-Type" "image/png"
                    assert_header "http://localhost:8080/img/javascript-40x40.svg" "Content-Type" "image/svg"
                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"

                    echo "🧪 Checking .md counterparts of pages that are excluded from llms.txt..."
                    # Pages excluded from the llms.txt index (see scripts/joinLlmsFiles.mjs)
                    # must still serve their .md markdown counterparts. Regression test for
                    # https://github.com/apify/apify-docs/pull/2470#discussion_r3161627392
                    assert_header "http://localhost:8080/sdk.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/open-source.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/v2/actor-builds-get.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/v2/dataset-get.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/academy/tutorials.md" "Content-Type" "text/markdown"

                    echo "🧪 Checking HTML alternate links to .md counterparts..."
                    # The HTML version of every page (including those excluded from the
                    # llms.txt index) must advertise its .md counterpart via
                    # <link rel="alternate" type="text/markdown" href="..."> so that
                    # AI crawlers can discover the markdown version.
                    assert_html_md_alternate "http://localhost:8080/sdk" "https://docs.apify.com/sdk.md"
                    assert_html_md_alternate "http://localhost:8080/open-source" "https://docs.apify.com/open-source.md"
                    assert_html_md_alternate "http://localhost:8080/api/v2/actor-builds-get" "https://docs.apify.com/api/v2/actor-builds-get.md"
                    assert_html_md_alternate "http://localhost:8080/api/v2/dataset-get" "https://docs.apify.com/api/v2/dataset-get.md"
                    assert_html_md_alternate "http://localhost:8080/academy/tutorials" "https://docs.apify.com/academy/tutorials.md"
                    # Sanity check: a regular (non-excluded) page also has the alternate.
                    assert_html_md_alternate "http://localhost:8080/platform/proxy/usage" "https://docs.apify.com/platform/proxy/usage.md"

                    echo "🧪 Checking Nginx responses... (apify-sdk-js)"
                    assert_final_content_type "http://localhost:8080/sdk/js" "text/html"
                    assert_final_content_type "http://localhost:8080/sdk/js" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/sdk/js/docs/introduction/quick-start" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/sdk/js/docs/introduction/quick-start.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/sdk/js/docs/introduction/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/sdk/js/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/sdk/js/llms-full.txt" "Content-Type" "text/markdown"

                    echo "🧪 Checking Nginx responses... (apify-sdk-python)"
                    assert_final_content_type "http://localhost:8080/sdk/python" "text/html"
                    assert_final_content_type "http://localhost:8080/sdk/python" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/sdk/python/docs/changelog" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/sdk/python/docs/changelog.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/sdk/python/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/sdk/python/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/sdk/python/llms-full.txt" "Content-Type" "text/markdown"

                    echo "🧪 Checking Nginx responses... (apify-client-js)"
                    assert_final_content_type "http://localhost:8080/api/client/js" "text/html"
                    assert_final_content_type "http://localhost:8080/api/client/js" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/api/client/js/docs/changelog" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/api/client/js/docs/changelog.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/client/js/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/api/client/js/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/client/js/llms-full.txt" "Content-Type" "text/markdown"

                    echo "🧪 Checking Nginx responses... (apify-client-python)"
                    assert_final_content_type "http://localhost:8080/api/client/python" "text/html"
                    assert_final_content_type "http://localhost:8080/api/client/python" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/api/client/python/docs/changelog" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/api/client/python/docs/changelog.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/client/python/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/api/client/python/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/api/client/python/llms-full.txt" "Content-Type" "text/markdown"

                    echo "🧪 Checking Nginx responses... (apify-cli)"
                    assert_final_content_type "http://localhost:8080/cli" "text/html"
                    assert_final_content_type "http://localhost:8080/cli" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/cli/docs/changelog" "Content-Type" "text/html"
                    assert_header "http://localhost:8080/cli/docs/changelog.md" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/cli/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
                    assert_header "http://localhost:8080/cli/llms.txt" "Content-Type" "text/markdown"
                    assert_header "http://localhost:8080/cli/llms-full.txt" "Content-Type" "text/markdown"

                    echo "✅ All Nginx header checks passed."

            -   name: Verify per-page .md navigation headers
                # Checks the Vercel-style nav header that scripts/addNavHeaders.mjs
                # prepends to every per-page .md in postbuild (#2596 / issue #2557).
                run: node scripts/checkNavHeaders.mjs http://localhost:8080

            -   name: Stop Nginx
                if: always()
                run: nginx -c "$(pwd)/default.conf" -s stop

    lint_content:
        name: Lint markdown content
        runs-on: ubuntu-latest
        steps:
            -   name: Checkout Source code
                uses: actions/checkout@v6

            -   name: Get changed files
                id: changed-files
                uses: tj-actions/changed-files@v47
                with:
                    files: '**/*.{md,mdx}'
                    files_ignore: '!sources/api/*.{md,mdx}'
                    separator: ","

            -   name: Use Node.js 24
                uses: actions/setup-node@v6
                with:
                    node-version: 24

            -   uses: apify/actions/pnpm-install@v1.1.2

            -   name: List and Lint Changed Markdown Files
                env:
                    ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
                run: |
                    IFS=',' read -ra FILE_ARRAY <<< "$ALL_CHANGED_FILES"
                    for file in "${FILE_ARRAY[@]}"; do
                      pnpm exec markdownlint "$file"
                    done


    lint_code:
        name: Lint app code
        runs-on: ubuntu-latest
        steps:
            -   name: Checkout Source code
                uses: actions/checkout@v6

            -   name: Use Node.js 24
                uses: actions/setup-node@v6
                with:
                    node-version: 24

            -   uses: apify/actions/pnpm-install@v1.1.2

            -   run: pnpm lint:code

            -   name: Format check
                run: pnpm format:check


    lint_images:
        name: Lint images
        runs-on: ubuntu-latest
        steps:
            -   name: Checkout Source code
                uses: actions/checkout@v6

            -   name: Get changed unoptimized image files in sources
                id: changed-files
                uses: tj-actions/changed-files@v47
                with:
                    files: |
                        sources/**/*.{png,jpg,jpeg,gif,bmp,tif,tiff,avif}
                    separator: "\n"

            -   name: Fail on unoptimized images
                if: steps.changed-files.outputs.any_changed == 'true'
                env:
                    UNOPTIMIZED_IMAGE_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
                run: |
                    echo "Unoptimized images detected! Please run the following commands:"
                    echo ""
                    while IFS= read -r file_path; do
                      echo "pnpm opt:images $file_path"
                    done <<< "$UNOPTIMIZED_IMAGE_FILES"
                    echo ""
                    echo "Don't forget to change image paths to end with .webp in your Markdown."
                    echo "If you point the same command at a whole directory, it will convert "
                    echo "and optimize all images inside."
                    exit 1